void swd::database::ensure_connection() {
boost::unique_lock<boost::mutex> scoped_lock(dbi_mutex_);
if (dbi_conn_ping(conn_) < 1) {
swd::log::i()->send(swd::notice, "Dropped database connection");
if (dbi_conn_connect(conn_) < 0) {
throw swd::exceptions::database_exception("Lost database connection");
}
}
}
static inline gboolean
afsql_dd_handle_insert_row_error_depending_on_connection_availability(AFSqlDestDriver *self,
LogMessage *msg,
LogPathOptions *path_options)
{
const gchar *dbi_error, *error_message;
if (dbi_conn_ping(self->dbi_ctx) == 1)
{
log_queue_push_head(self->queue, msg, path_options);
return TRUE;
}
if (afsql_dd_is_transaction_handling_enabled(self))
{
error_message = "SQL connection lost in the middle of a transaction,"
" rewinding backlog and starting again";
afsql_dd_handle_transaction_error(self);
}
else
{
error_message = "Error, no SQL connection after failed query attempt";
log_queue_push_head(self->queue, msg, path_options);
}
dbi_conn_error(self->dbi_ctx, &dbi_error);
msg_error(error_message,
evt_tag_str("type", self->type),
evt_tag_str("host", self->host),
evt_tag_str("port", self->port),
evt_tag_str("username", self->user),
evt_tag_str("database", self->database),
evt_tag_str("error", dbi_error),
NULL);
return FALSE;
}
static gboolean
afsql_dd_insert_fail_handler(AFSqlDestDriver *self, LogMessage *msg,
LogPathOptions *path_options)
{
if (self->failed_message_counter < self->num_retries - 1)
{
log_queue_push_head(self->queue, msg, path_options);
/* database connection status sanity check after failed query */
if (dbi_conn_ping(self->dbi_ctx) != 1)
{
const gchar *dbi_error;
dbi_conn_error(self->dbi_ctx, &dbi_error);
msg_error("Error, no SQL connection after failed query attempt",
evt_tag_str("type", self->type),
evt_tag_str("host", self->host),
evt_tag_str("port", self->port),
evt_tag_str("username", self->user),
evt_tag_str("database", self->database),
evt_tag_str("error", dbi_error),
NULL);
return FALSE;
}
self->failed_message_counter++;
return FALSE;
}
msg_error("Multiple failures while inserting this record into the database, message dropped",
evt_tag_int("attempts", self->num_retries),
NULL);
stats_counter_inc(self->dropped_messages);
log_msg_drop(msg, path_options);
self->failed_message_counter = 0;
return TRUE;
}
static int cdbi_connect_database (cdbi_database_t *db) /* {{{ */
{
dbi_driver driver;
dbi_conn connection;
size_t i;
int status;
if (db->connection != NULL)
{
status = dbi_conn_ping (db->connection);
if (status != 0) /* connection is alive */
return (0);
dbi_conn_close (db->connection);
db->connection = NULL;
}
driver = dbi_driver_open_r (db->driver, dbi_instance);
if (driver == NULL)
{
ERROR ("dbi plugin: cdbi_connect_database: dbi_driver_open_r (%s) failed.",
db->driver);
INFO ("dbi plugin: Maybe the driver isn't installed? "
"Known drivers are:");
for (driver = dbi_driver_list_r (NULL, dbi_instance);
driver != NULL;
driver = dbi_driver_list_r (driver, dbi_instance))
{
INFO ("dbi plugin: * %s", dbi_driver_get_name (driver));
}
return (-1);
}
connection = dbi_conn_open (driver);
if (connection == NULL)
{
ERROR ("dbi plugin: cdbi_connect_database: dbi_conn_open (%s) failed.",
db->driver);
return (-1);
}
/* Set all the driver options. Because this is a very very very generic
* interface, the error handling is kind of long. If an invalid option is
* encountered, it will get a list of options understood by the driver and
* report that as `INFO'. This way, users hopefully don't have too much
* trouble finding out how to configure the plugin correctly.. */
for (i = 0; i < db->driver_options_num; i++)
{
if (db->driver_options[i].is_numeric)
{
status = dbi_conn_set_option_numeric (connection,
db->driver_options[i].key, db->driver_options[i].value.numeric);
if (status != 0)
{
char errbuf[1024];
ERROR ("dbi plugin: cdbi_connect_database (%s): "
"dbi_conn_set_option_numeric (\"%s\", %i) failed: %s.",
db->name,
db->driver_options[i].key, db->driver_options[i].value.numeric,
cdbi_strerror (connection, errbuf, sizeof (errbuf)));
}
}
else
{
status = dbi_conn_set_option (connection,
db->driver_options[i].key, db->driver_options[i].value.string);
if (status != 0)
{
char errbuf[1024];
ERROR ("dbi plugin: cdbi_connect_database (%s): "
"dbi_conn_set_option (\"%s\", \"%s\") failed: %s.",
db->name,
db->driver_options[i].key, db->driver_options[i].value.string,
cdbi_strerror (connection, errbuf, sizeof (errbuf)));
}
}
if (status != 0)
{
char const *opt;
INFO ("dbi plugin: This is a list of all options understood "
"by the `%s' driver:", db->driver);
for (opt = dbi_conn_get_option_list (connection, NULL);
opt != NULL;
opt = dbi_conn_get_option_list (connection, opt))
{
INFO ("dbi plugin: * %s", opt);
}
dbi_conn_close (connection);
return (-1);
}
} /* for (i = 0; i < db->driver_options_num; i++) */
status = dbi_conn_connect (connection);
if (status != 0)
{
char errbuf[1024];
ERROR ("dbi plugin: cdbi_connect_database (%s): "
"dbi_conn_connect failed: %s",
db->name, cdbi_strerror (connection, errbuf, sizeof (errbuf)));
dbi_conn_close (connection);
return (-1);
}
if (db->select_db != NULL)
{
status = dbi_conn_select_db (connection, db->select_db);
if (status != 0)
{
char errbuf[1024];
WARNING ("dbi plugin: cdbi_connect_database (%s): "
"dbi_conn_select_db (%s) failed: %s. Check the `SelectDB' option.",
db->name, db->select_db,
cdbi_strerror (connection, errbuf, sizeof (errbuf)));
dbi_conn_close (connection);
return (-1);
}
}
db->connection = connection;
return (0);
} /* }}} int cdbi_connect_database */
dbi_result _db_rawquery(const char *file, int line, database *db, int log_dupes, char *qry)
{
dbi_result result;
const char *errmsg;
int tries = 0;
if (debug > 4) {
LOGRAW(LOG_DEBUG, qry);
}
if (!dbi_conn_ping(db->conn)) {
if (db->conn) dbi_conn_close(db->conn);
db->conn = dbi_conn_new(db->driver);
if (db->conn == NULL) {
perror(db->driver);
exit(1);
}
dbi_conn_set_option(db->conn, "host", db->host);
dbi_conn_set_option(db->conn, "port", db->port);
dbi_conn_set_option(db->conn, "username", db->username);
dbi_conn_set_option(db->conn, "password", db->password);
dbi_conn_set_option(db->conn, "dbname", db->name);
retry:
if (++tries == 3) exit(1);
if (dbi_conn_connect(db->conn) < 0) {
dbi_conn_error(db->conn, &errmsg);
_logsrce = file;
_logline = line;
_LOG(LOG_ERR, "%s connection failed to %s:%s:%s %s", db->driver, db->host, db->port, db->name, errmsg);
sleep(3);
goto retry;
}
{
char versionstring[VERSIONSTRING_LENGTH];
dbi_driver driver;
driver = dbi_conn_get_driver(db->conn);
dbi_conn_get_engine_version_string(db->conn, versionstring);
LOG(LOG_INFO, "using driver: %s, version %s, compiled %s", dbi_driver_get_filename(driver), dbi_driver_get_version(driver),
dbi_driver_get_date_compiled(driver));
LOG(LOG_INFO, "connected to %s:%s:%s, server version %s", db->host, db->port, db->name, versionstring);
}
}
if (debug > 2) {
char *src, *dst, buf[4096];
for (dst = buf, src = qry; *src; src++, dst++) {
if (*src == '%') *dst++ = '%';
*dst = *src;
}
*dst = 0;
LOG(LOG_INFO, buf);
}
result = dbi_conn_query(db->conn, qry);
if (result == NULL) {
int ret = dbi_conn_error(db->conn, &errmsg);
_logsrce = file;
_logline = line;
_LOG(LOG_ERR, "query failed: %s (%s)", errmsg, qry);
if (ret == DBI_ERROR_NOCONN) {
dbi_conn_close(db->conn);
db->conn = NULL;
sleep(3);
goto retry;
}
}
return(result);
}
/**
* Prepare SQL queries and perform them.
*
* @param format printf-like string of SQL query
* @param callback pointer to function to be called on results of SQL query
* @param firstarg pointer to buffer for callback returned data
* @return 0 (always, due to SQLite policy)
*/
int tagsistant_real_query(
dbi_conn dbi,
const char *format,
int (*callback)(void *, dbi_result),
char *file,
int line,
void *firstarg,
...)
{
va_list ap;
va_start(ap, firstarg);
/* check if connection has been created */
if (NULL == dbi) {
dbg('s', LOG_ERR, "ERROR! DBI connection was not initialized!");
return(0);
}
#if TAGSISTANT_USE_QUERY_MUTEX
/* lock the connection mutex */
g_mutex_lock(&tagsistant_query_mutex);
#endif
/* check if the connection is alive */
if (!dbi_conn_ping(dbi) && dbi_conn_connect(dbi) < 0) {
#if TAGSISTANT_USE_QUERY_MUTEX
g_mutex_unlock(&tagsistant_query_mutex);
#endif
dbg('s', LOG_ERR, "ERROR! DBI Connection has gone!");
return(0);
}
/* replace all the single or double quotes with "<><>" in the format */
gchar *escaped_format = g_regex_replace_literal(RX1, format, -1, 0, "<><>", 0, NULL);
/* format the statement */
gchar *statement = g_strdup_vprintf(escaped_format, ap);
if (NULL == statement) {
#if TAGSISTANT_USE_QUERY_MUTEX
/* lock the connection mutex */
g_mutex_unlock(&tagsistant_query_mutex);
#endif
dbg('s', LOG_ERR, "Null SQL statement");
g_free(escaped_format);
return(0);
}
/* prepend a backslash to all the single quotes inside the arguments */
gchar *escaped_statement_tmp = g_regex_replace_literal(RX2, statement, -1, 0, "''", 0, NULL);
/* replace "<><>" with a single quote */
gchar *escaped_statement = g_regex_replace_literal(RX3, escaped_statement_tmp, -1, 0, "'", 0, NULL);
/* log and do the query */
dbg('s', LOG_INFO, "SQL from %s:%d: [%s]", file, line, escaped_statement);
dbi_result result = dbi_conn_query(dbi, escaped_statement);
tagsistant_dirty_logging(escaped_statement);
tagsistant_wal(dbi, escaped_statement);
g_free_null(escaped_format);
g_free_null(escaped_statement_tmp);
g_free_null(escaped_statement);
/* call the callback function on results or report an error */
int rows = 0;
if (result) {
if (callback) {
while (dbi_result_next_row(result)) {
callback(firstarg, result);
rows++;
}
}
dbi_result_free(result);
} else {
/* get the error message */
const char *errmsg = NULL;
dbi_conn_error(dbi, &errmsg);
if (errmsg) dbg('s', LOG_ERR, "Error: %s.", errmsg);
}
#if TAGSISTANT_USE_QUERY_MUTEX
g_mutex_unlock(&tagsistant_query_mutex);
#endif
return(rows);
}
/**
* Parse command line options, create connection object,
* start the connection and finally create database schema
*
* @return DBI connection handle
*/
dbi_conn *tagsistant_db_connection(int start_transaction)
{
/* DBI connection handler used by subsequent calls to dbi_* functions */
dbi_conn dbi = NULL;
if (start_transaction) {
g_rw_lock_writer_lock(&(tagsistant_query_rwlock));
} else {
g_rw_lock_reader_lock(&(tagsistant_query_rwlock));
}
/* lock the pool */
g_mutex_lock(&tagsistant_connection_pool_lock);
GList *pool = tagsistant_connection_pool;
while (pool) {
dbi = (dbi_conn) pool->data;
/* check if the connection is still alive */
if (!dbi_conn_ping(dbi) && dbi_conn_connect(dbi) < 0) {
dbi_conn_close(dbi);
tagsistant_connection_pool = g_list_delete_link(tagsistant_connection_pool, pool);
connections--;
} else {
tagsistant_connection_pool = g_list_remove_link(tagsistant_connection_pool, pool);
g_list_free_1(pool);
break;
}
pool = pool->next;
}
/*
* unlock the pool mutex only if the backend is not SQLite
*/
g_mutex_unlock(&tagsistant_connection_pool_lock);
if (!dbi) {
// initialize DBI drivers
if (TAGSISTANT_DBI_MYSQL_BACKEND == dboptions.backend) {
if (!tagsistant_driver_is_available("mysql")) {
fprintf(stderr, "MySQL driver not installed\n");
dbg('s', LOG_ERR, "MySQL driver not installed");
exit (1);
}
// unlucky, MySQL does not provide INTERSECT operator
tagsistant.sql_backend_have_intersect = 0;
// create connection
#if TAGSISTANT_REENTRANT_DBI
dbi = dbi_conn_new_r("mysql", tagsistant.dbi_instance);
#else
dbi = dbi_conn_new("mysql");
#endif
if (NULL == dbi) {
dbg('s', LOG_ERR, "Error creating MySQL connection");
exit (1);
}
// set connection options
dbi_conn_set_option(dbi, "host", dboptions.host);
dbi_conn_set_option(dbi, "dbname", dboptions.db);
dbi_conn_set_option(dbi, "username", dboptions.username);
dbi_conn_set_option(dbi, "password", dboptions.password);
dbi_conn_set_option(dbi, "encoding", "UTF-8");
} else if (TAGSISTANT_DBI_SQLITE_BACKEND == dboptions.backend) {
if (!tagsistant_driver_is_available("sqlite3")) {
fprintf(stderr, "SQLite3 driver not installed\n");
dbg('s', LOG_ERR, "SQLite3 driver not installed");
exit(1);
}
// create connection
#if TAGSISTANT_REENTRANT_DBI
dbi = dbi_conn_new_r("sqlite3", tagsistant.dbi_instance);
#else
dbi = dbi_conn_new("sqlite3");
#endif
if (NULL == dbi) {
dbg('s', LOG_ERR, "Error connecting to SQLite3");
exit (1);
}
// set connection options
dbi_conn_set_option(dbi, "dbname", "tags.sql");
dbi_conn_set_option(dbi, "sqlite3_dbdir", tagsistant.repository);
} else {
dbg('s', LOG_ERR, "No or wrong database family specified!");
exit (1);
}
// try to connect
if (dbi_conn_connect(dbi) < 0) {
int error = dbi_conn_error(dbi, NULL);
dbg('s', LOG_ERR, "Could not connect to DB (error %d). Please check the --db settings", error);
exit(1);
}
connections++;
dbg('s', LOG_INFO, "SQL connection established");
}
/* start a transaction */
if (start_transaction) {
#if TAGSISTANT_USE_INTERNAL_TRANSACTIONS
switch (tagsistant.sql_database_driver) {
case TAGSISTANT_DBI_SQLITE_BACKEND:
tagsistant_query("begin transaction", dbi, NULL, NULL);
break;
case TAGSISTANT_DBI_MYSQL_BACKEND:
tagsistant_query("start transaction", dbi, NULL, NULL);
break;
}
#else
dbi_conn_transaction_begin(dbi);
#endif
}
return(dbi);
}
void do_db( struct configuration_data *config, char *dbstring)
{
int rc;
int try;
const char *error;
dbi_result result;
/**
* a NULL dbstring? May happen when a R/W VFS function
* hat 0 bytes to transfer. This data is not relevant
* for statistics.
*/
if (dbstring == NULL) return;
/**
* Check if the connection is alive. We try ten times
* to restore the connection if not
*/
for (try = 0; try < 10; try++) {
rc = dbi_conn_ping( config->DBIconn );
if (rc == 1) {
result = dbi_conn_query(config->DBIconn, dbstring);
if (result == NULL) {
dbi_conn_error(config->DBIconn,&error);
syslog(LOG_DEBUG,"DBI ERROR: %s",error);
rc = 0;}
else break;
}
}
if (rc == 0 ) {
syslog(LOG_DEBUG, "ERROR: Database handling error!");
return;
}
dbi_result_free(result);
}
void cleanup_cache( TALLOC_CTX *ctx,struct configuration_data *config,
struct cache_entry *entry)
{
char *dbstring;
struct cache_entry *go = entry;
struct cache_entry *backup;
struct cache_entry *backup2 = NULL;
struct cache_entry *down =NULL;
// left
go = go->left;
while (go != NULL) {
backup = go->left;
dbstring = cache_create_database_string(ctx,go,config);
do_db(config,dbstring);
talloc_free(dbstring);
// go down
down = go->down;
while (down != NULL) {
backup2 = down->down;
dbstring = cache_create_database_string(ctx,down,config);
do_db(config,dbstring);
talloc_free(dbstring);
talloc_free(down);
down = backup2;
}
talloc_free(go);
go = backup;
}
// right
go = entry->right;
while (go != NULL) {
backup = go->right;
dbstring = cache_create_database_string(ctx,go,config);
do_db(config,dbstring);
talloc_free(dbstring);
// go down
down = go->down;
while (down != NULL) {
backup2 = down->down;
dbstring = cache_create_database_string(ctx,down,config);
do_db(config,dbstring);
talloc_free(dbstring);
talloc_free(down);
down = backup2;
}
talloc_free(go);
go = backup;
}
// other_ops
go = entry->other_ops;
while (go != NULL) {
backup = go->other_ops;
dbstring = cache_create_database_string(ctx,go,config);
do_db(config,dbstring);
talloc_free(dbstring);
talloc_free(go);
go = backup;
}
// delete tree begin
dbstring = cache_create_database_string(ctx,entry,config);
do_db(config,dbstring);
if (dbstring != NULL) talloc_free(dbstring);
talloc_free(entry);
}
/*
* cache_manager runs as a detached thread. Every half second,
* it's checking if there's data available to store to the DB.
* in case of yes, it walks through the list of data, enables
* a new cache, and stores the data in the DB
*/
void cache_manager(struct configuration_data *config )
{
int maintenance_c_val;
if (config->precision>0)
maintenance_c_val = config->maintenance_seconds / config->precision;
else maintenance_c_val = config->maintenance_seconds;
int maintenance_count = 0;
pthread_detach(pthread_self());
/* backup the start and make it possible to
* allocate a new cache beginning
*/
while (1 == 1) {
if (config->precision != 0) sleep(config->precision);
else sleep(5);
maintenance_count++;
pthread_mutex_lock(&cache_mutex);
struct cache_entry *backup = cache_start;
TALLOC_CTX *dbpool = talloc_pool(NULL,2048);
cache_start = NULL;
cache_end = NULL;
pthread_mutex_unlock(&cache_mutex);
if (backup != NULL) {
/* store all existing entries into the database */
do_db(config,"BEGIN TRANSACTION;");
cleanup_cache( dbpool,config,backup);
do_db(config,"COMMIT;");
}
talloc_free(dbpool);
if (maintenance_count == maintenance_c_val) {
char String[400];
char dbstring[300];
struct tm *tm;
do_db(config,"BEGIN TRANSACTION;");
time_t today=time(NULL);
time_t delete_date=today - config->maint_run_time;
tm = localtime ( &delete_date );
sprintf(String,"%04d-%02d-%02d %02d:%02d:%02d", \
tm->tm_year+1900, tm->tm_mon+1, tm->tm_mday, \
tm->tm_hour, tm->tm_min, tm->tm_sec);
strcpy(dbstring,"delete from data where timestamp < '");
strcat(dbstring,String);
strcat(dbstring,"';");
do_db(config,dbstring);
do_db(config,"COMMIT;");
maintenance_count = 0;
}
}
}
