示例#1
0
/**
 * afsql_dd_insert_db:
 *
 * This function is running in the database thread
 *
 * Returns: FALSE to indicate that the connection should be closed and
 * this destination suspended for time_reopen() time.
 **/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
  GString *table, *query_string;
  LogMessage *msg;
  gboolean success;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afsql_dd_connect(self);

  success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  table = afsql_dd_validate_table(self, msg);
  if (!table)
    {
      /* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
      msg_error("Error checking table, disconnecting from database, trying again shortly",
                evt_tag_int("time_reopen", self->time_reopen),
                NULL);
      msg_set_context(NULL);
      g_string_free(table, TRUE);
      return afsql_dd_insert_fail_handler(self, msg, &path_options);
    }

  query_string = afsql_dd_construct_query(self, table, msg);

  if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
    return FALSE;

  success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
  if (success && self->flush_lines_queued != -1)
    {
      self->flush_lines_queued++;

      if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self))
        return FALSE;
    }

  g_string_free(table, TRUE);
  g_string_free(query_string, TRUE);

  msg_set_context(NULL);

  if (!success)
    return afsql_dd_insert_fail_handler(self, msg, &path_options);

  /* we only ACK if each INSERT is a separate transaction */
  if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
    log_msg_ack(msg, &path_options);
  log_msg_unref(msg);
  step_sequence_number(&self->seq_num);
  self->failed_message_counter = 0;

  return TRUE;
}
示例#2
0
/**
 * afsql_dd_database_thread:
 *
 * This is the thread inserting records into the database.
 **/
static void
afsql_dd_database_thread(gpointer arg)
{
  AFSqlDestDriver *self = (AFSqlDestDriver *) arg;

  msg_verbose("Database thread started",
              evt_tag_str("driver", self->super.super.id),
              NULL);
  while (!self->db_thread_terminate)
    {
      g_mutex_lock(self->db_thread_mutex);
      if (self->db_thread_suspended)
        {
          afsql_dd_wait_for_suspension_wakeup(self);
          /* we loop back to check if the thread was requested to terminate */
        }
      else if (!log_queue_check_items(self->queue, NULL, afsql_dd_message_became_available_in_the_queue, self, NULL))
        {
          /* we have nothing to INSERT into the database, let's wait we get some new stuff */

          if (self->flush_lines_queued > 0)
            {
              if (!afsql_dd_commit_txn(self))
                {
                  afsql_dd_disconnect(self);
                  afsql_dd_suspend(self);
                  g_mutex_unlock(self->db_thread_mutex);
                  continue;
                }
            }
          else if (!self->db_thread_terminate)
            {
              g_cond_wait(self->db_thread_wakeup_cond, self->db_thread_mutex);
            }

          /* we loop back to check if the thread was requested to terminate */
        }
      g_mutex_unlock(self->db_thread_mutex);

      if (self->db_thread_terminate)
        break;

      if (!afsql_dd_insert_db(self))
        {
          afsql_dd_disconnect(self);
          afsql_dd_suspend(self);
        }
    }

  while (log_queue_get_length(self->queue) > 0)
    {
      if (!afsql_dd_insert_db(self))
        {
          goto exit;
        }
    }

  if (self->flush_lines_queued > 0)
    {
      /* we can't do anything with the return value here. if commit isn't
       * successful, we get our backlog back, but we have no chance
       * submitting that back to the SQL engine.
       */

      afsql_dd_commit_txn(self);
    }
exit:
  afsql_dd_disconnect(self);

  msg_verbose("Database thread finished",
              evt_tag_str("driver", self->super.super.id),
              NULL);
}
示例#3
0
/**
 * afsql_dd_insert_db:
 *
 * This function is running in the database thread
 *
 * Returns: FALSE to indicate that the connection should be closed and
 * this destination suspended for time_reopen() time.
 **/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
  GString *table = NULL;
  GString *insert_command = NULL;
  LogMessage *msg;
  gboolean success;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  if (!afsql_dd_ensure_initialized_connection(self))
    return FALSE;

  /* connection established, try to insert a message */
  success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, self->flags & AFSQL_DDF_EXPLICIT_COMMITS);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  table = afsql_dd_ensure_accessible_database_table(self, msg);

  if (!table)
    {
      success = FALSE;
      goto out;
    }

  if (afsql_dd_should_start_new_transaction(self) && !afsql_dd_begin_txn(self))
    {
      success = FALSE;
      goto out;
    }

  insert_command = afsql_dd_build_insert_command(self, msg, table);
  success = afsql_dd_run_query(self, insert_command->str, FALSE, NULL);

  if (success && self->flush_lines_queued != -1)
    {
      self->flush_lines_queued++;

      if (afsql_dd_should_commit_transaction(self) && !afsql_dd_commit_txn(self))
        {
          /* Assuming that in case of error, the queue is rewound by afsql_dd_commit_txn() */

          g_string_free(insert_command, TRUE);
          msg_set_context(NULL);

          return FALSE;
        }
    }

 out:

  if (table != NULL)
    g_string_free(table, TRUE);

  if (insert_command != NULL)
    g_string_free(insert_command, TRUE);

  msg_set_context(NULL);

  if (success)
    {
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
      step_sequence_number(&self->seq_num);
      self->failed_message_counter = 0;
    }
  else
    {
      if (self->failed_message_counter < self->num_retries - 1)
        {
          if (!afsql_dd_handle_insert_row_error_depending_on_connection_availability(self, msg, &path_options))
            return FALSE;

          self->failed_message_counter++;
        }
      else
        {
          msg_error("Multiple failures while inserting this record into the database, message dropped",
                    evt_tag_int("attempts", self->num_retries),
                    NULL);
          stats_counter_inc(self->dropped_messages);
          log_msg_drop(msg, &path_options);
          self->failed_message_counter = 0;
          success = TRUE;
        }
    }

  return success;
}
示例#4
0
/**
 * afsql_dd_database_thread:
 *
 * This is the thread inserting records into the database.
 **/
static gpointer
afsql_dd_database_thread(gpointer arg)
{
  AFSqlDestDriver *self = (AFSqlDestDriver *) arg;

  msg_verbose("Database thread started",
              evt_tag_str("driver", self->super.super.id),
              NULL);
  while (!self->db_thread_terminate)
    {
      g_mutex_lock(self->db_thread_mutex);
      if (self->db_thread_suspended)
        {
          /* we got suspended, probably because of a connection error,
           * during this time we only get wakeups if we need to be
           * terminated. */
          if (!self->db_thread_terminate)
            g_cond_timed_wait(self->db_thread_wakeup_cond, self->db_thread_mutex, &self->db_thread_suspend_target);
          self->db_thread_suspended = FALSE;
          g_mutex_unlock(self->db_thread_mutex);

          /* we loop back to check if the thread was requested to terminate */
        }
      else if (log_queue_get_length(self->queue) == 0)
        {
          /* we have nothing to INSERT into the database, let's wait we get some new stuff */

          if (self->flush_lines_queued > 0 && self->flush_timeout > 0)
            {
              GTimeVal flush_target;

              g_get_current_time(&flush_target);
              g_time_val_add(&flush_target, self->flush_timeout * 1000);
              if (!self->db_thread_terminate && !g_cond_timed_wait(self->db_thread_wakeup_cond, self->db_thread_mutex, &flush_target))
                {
                  /* timeout elapsed */
                  if (!afsql_dd_commit_txn(self, FALSE))
                    {
                      afsql_dd_disconnect(self);
                      afsql_dd_suspend(self);
                      g_mutex_unlock(self->db_thread_mutex);
                      continue;
                    }
                }
            }
          else if (!self->db_thread_terminate)
            {
              g_cond_wait(self->db_thread_wakeup_cond, self->db_thread_mutex);
            }
          g_mutex_unlock(self->db_thread_mutex);

          /* we loop back to check if the thread was requested to terminate */
        }
      else
        g_mutex_unlock(self->db_thread_mutex);

      if (self->db_thread_terminate)
        break;

      if (!afsql_dd_insert_db(self))
        {
          afsql_dd_disconnect(self);
          afsql_dd_suspend(self);
        }
    }
  if (self->flush_lines_queued > 0)
    {
      /* we can't do anything with the return value here. if commit isn't
       * successful, we get our backlog back, but we have no chance
       * submitting that back to the SQL engine.
       */

      afsql_dd_commit_txn(self, TRUE);
    }

  afsql_dd_disconnect(self);

  msg_verbose("Database thread finished",
              evt_tag_str("driver", self->super.super.id),
              NULL);
  return NULL;
}
示例#5
0
/**
 * afsql_dd_insert_db:
 *
 * This function is running in the database thread
 *
 * Returns: FALSE to indicate that the connection should be closed and
 * this destination suspended for time_reopen() time.
 **/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
  GString *table, *query_string;
  LogMessage *msg;
  gboolean success;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afsql_dd_connect(self);

  g_mutex_lock(self->db_thread_mutex);

  /* FIXME: this is a workaround because of the non-proper locking semantics
   * of the LogQueue.  It might happen that the _queue() method sees 0
   * elements in the queue, while the thread is still busy processing the
   * previous message.  In that case arming the parallel push callback is
   * not needed and will cause assertions to fail.  This is ugly and should
   * be fixed by properly defining the "blocking" semantics of the LogQueue
   * object w/o having to rely on user-code messing with parallel push
   * callbacks. */
  log_queue_reset_parallel_push(self->queue);
  success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
  g_mutex_unlock(self->db_thread_mutex);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  table = afsql_dd_validate_table(self, msg);
  if (!table)
    {
      /* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
      msg_error("Error checking table, disconnecting from database, trying again shortly",
                evt_tag_int("time_reopen", self->time_reopen),
                NULL);
      msg_set_context(NULL);
      g_string_free(table, TRUE);
      return afsql_dd_insert_fail_handler(self, msg, &path_options);
    }

  query_string = afsql_dd_construct_query(self, table, msg);

  if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
    return FALSE;

  success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
  if (success && self->flush_lines_queued != -1)
    {
      self->flush_lines_queued++;

      if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE))
        return FALSE;
    }

  g_string_free(table, TRUE);
  g_string_free(query_string, TRUE);

  msg_set_context(NULL);

  if (!success)
    return afsql_dd_insert_fail_handler(self, msg, &path_options);

  /* we only ACK if each INSERT is a separate transaction */
  if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
    log_msg_ack(msg, &path_options);
  log_msg_unref(msg);
  step_sequence_number(&self->seq_num);
  self->failed_message_counter = 0;

  return TRUE;
}