static void test_rewind_over_eof(LogQueueDiskReliable *dq) { LogMessage *msg3 = log_msg_new_mark(); LogMessage *read_message3; LogPathOptions local_options = LOG_PATH_OPTIONS_INIT; msg3->ack_func = _dummy_ack; log_queue_push_tail(&dq->super.super, msg3, &local_options); gint64 previous_read_head = dq->super.qdisk->hdr->read_head; read_message3 = log_queue_pop_head(&dq->super.super, &local_options); assert_true(read_message3 != NULL, ASSERTION_ERROR("Can't read message from queue")); assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position")); assert_true(msg3 == read_message3, ASSERTION_ERROR("Message 3 isn't read from qreliable")); log_msg_unref(read_message3); log_queue_rewind_backlog(&dq->super.super, 1); assert_gint(dq->super.qdisk->hdr->read_head, previous_read_head, ASSERTION_ERROR("Read head is corrupted")); read_message3 = log_queue_pop_head(&dq->super.super, &local_options); assert_true(read_message3 != NULL, ASSERTION_ERROR("Can't read message from queue")); assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position")); assert_true(msg3 == read_message3, ASSERTION_ERROR("Message 3 isn't read from qreliable")); log_msg_drop(msg3, &local_options, AT_PROCESSED); }
/* * The method make the following situation * the backlog contains 6 messages * the qbacklog contains 3 messages, * but messages in qbacklog are the end of the backlog */ void _prepare_rewind_backlog_test(LogQueueDiskReliable *dq, gint64 *start_pos) { gint i; for (i = 0; i < 8; i++) { LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; LogMessage *mark_message; mark_message = log_msg_new_mark(); mark_message->ack_func = _dummy_ack; log_queue_push_tail(&dq->super.super, mark_message, &path_options); } /* Lets read the messages and leave them in the backlog */ for (i = 0; i < 8; i++) { LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; msg = log_queue_pop_head(&dq->super.super, &path_options); log_msg_unref(msg); } /* Ack the messages which are not in the qbacklog */ log_queue_ack_backlog(&dq->super.super, 5); assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(3), ASSERTION_ERROR("Incorrect number of items in the qbacklog")); *start_pos = dq->super.qdisk->hdr->read_head; /* Now write 3 more messages and read them from buffer * the number of messages in the qbacklog should not be changed * The backlog should contain 6 messages * from these 6 messages 3 messages are cached in the qbacklog * No readable messages are in the queue */ for (i = 0; i < 3; i++) { LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; LogMessage *mark_message; mark_message = log_msg_new_mark(); mark_message->ack_func = _dummy_ack; log_queue_push_tail(&dq->super.super, mark_message, &path_options); mark_message = log_queue_pop_head(&dq->super.super, &path_options); assert_gint(dq->qreliable->length, 0, ASSERTION_ERROR("Incorrect number of items in the qreliable")); assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(3), ASSERTION_ERROR("Incorrect number of items in the qbacklog")); log_msg_unref(mark_message); } assert_gint(dq->super.qdisk->hdr->backlog_len, 6, ASSERTION_ERROR("Incorrect number of messages in the backlog")); assert_gint(dq->super.qdisk->hdr->length, 0, ASSERTION_ERROR("Reliable diskq isn't empty")); }
static gboolean afamqp_worker_insert(LogThrDestDriver *s) { AMQPDestDriver *self = (AMQPDestDriver *)s; gboolean success; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; afamqp_dd_connect(self, TRUE); success = log_queue_pop_head(s->queue, &msg, &path_options, FALSE, FALSE); if (!success) return TRUE; msg_set_context(msg); success = afamqp_worker_publish (self, msg); msg_set_context(NULL); if (success) { stats_counter_inc(s->stored_messages); step_sequence_number(&self->seq_num); log_msg_ack(msg, &path_options); log_msg_unref(msg); } else log_queue_push_head(s->queue, msg, &path_options); return success; }
/* runs in a dedicated thread */ static LogThreadedFetchResult _fetch(LogThreadedFetcherDriver *s) { ThreadedDiskqSourceDriver *self = (ThreadedDiskqSourceDriver *) s; LogPathOptions local_options = LOG_PATH_OPTIONS_INIT; gint64 remaining_messages = log_queue_get_length(self->queue); LogMessage *msg = log_queue_pop_head(self->queue, &local_options); if (!msg) { if (remaining_messages != 0) msg_error("Closing corrupt diskq file, waiting for new", evt_tag_long("lost_messages", remaining_messages), evt_tag_str("file", self->filename)); else msg_info("Diskq file has been read, waiting for new file", evt_tag_str("file", self->filename)); _close_diskq(s); self->waiting_for_file_change = TRUE; LogThreadedFetchResult result = { THREADED_FETCH_NOT_CONNECTED, NULL }; return result; } LogThreadedFetchResult result = { THREADED_FETCH_SUCCESS, msg }; return result; }
gpointer threaded_consume(gpointer st) { LogQueue *q = (LogQueue *) st; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; gboolean success; gint i; for (i = 0; i < 100000; i++) { g_static_mutex_lock(&threaded_lock); msg = NULL; success = log_queue_pop_head(q, &msg, &path_options, FALSE); g_static_mutex_unlock(&threaded_lock); g_assert(!success || (success && msg != NULL)); if (!success) { fprintf(stderr, "Queue didn't return enough messages: i=%d\n", i); return GUINT_TO_POINTER(1); } log_msg_ack(msg, &path_options); log_msg_unref(msg); } return NULL; }
/** * afsql_dd_insert_db: * * This function is running in the database thread * * Returns: FALSE to indicate that the connection should be closed and * this destination suspended for time_reopen() time. **/ static gboolean afsql_dd_insert_db(AFSqlDestDriver *self) { GString *table, *query_string; LogMessage *msg; gboolean success; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; afsql_dd_connect(self); success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE); if (!success) return TRUE; msg_set_context(msg); table = afsql_dd_validate_table(self, msg); if (!table) { /* If validate table is FALSE then close the connection and wait time_reopen time (next call) */ msg_error("Error checking table, disconnecting from database, trying again shortly", evt_tag_int("time_reopen", self->time_reopen), NULL); msg_set_context(NULL); g_string_free(table, TRUE); return afsql_dd_insert_fail_handler(self, msg, &path_options); } query_string = afsql_dd_construct_query(self, table, msg); if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self)) return FALSE; success = afsql_dd_run_query(self, query_string->str, FALSE, NULL); if (success && self->flush_lines_queued != -1) { self->flush_lines_queued++; if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self)) return FALSE; } g_string_free(table, TRUE); g_string_free(query_string, TRUE); msg_set_context(NULL); if (!success) return afsql_dd_insert_fail_handler(self, msg, &path_options); /* we only ACK if each INSERT is a separate transaction */ if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0) log_msg_ack(msg, &path_options); log_msg_unref(msg); step_sequence_number(&self->seq_num); self->failed_message_counter = 0; return TRUE; }
static void test_read_over_eof(LogQueueDiskReliable *dq, LogMessage *msg1, LogMessage *msg2) { LogPathOptions read_options; LogMessage *read_message1; LogMessage *read_message2; read_message1 = log_queue_pop_head(&dq->super.super, &read_options); assert_true(read_message1 != NULL, ASSERTION_ERROR("Can't read message from queue")); read_message2 = log_queue_pop_head(&dq->super.super, &read_options); assert_true(read_message2 != NULL, ASSERTION_ERROR("Can't read message from queue")); assert_gint(dq->qreliable->length, 0, ASSERTION_ERROR("Queue reliable isn't empty")); assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(2), ASSERTION_ERROR("Messages aren't in the qbacklog")); assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position")); assert_true(msg1 == read_message1, ASSERTION_ERROR("Message 1 isn't read from qreliable")); assert_true(msg2 == read_message2, ASSERTION_ERROR("Message 2 isn't read from qreliable")); }
static gboolean afmongodb_worker_insert (MongoDBDestDriver *self) { gboolean success; guint8 *oid; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; afmongodb_dd_connect(self, TRUE); success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE); if (!success) return TRUE; msg_set_context(msg); bson_reset (self->bson); oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num); bson_append_oid (self->bson, "_id", oid); g_free (oid); value_pairs_walk(self->vp, afmongodb_vp_obj_start, afmongodb_vp_process_value, afmongodb_vp_obj_end, msg, self->seq_num, self->bson); bson_finish (self->bson); if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1, (const bson **)&self->bson)) { msg_error("Network error while inserting into MongoDB", evt_tag_int("time_reopen", self->time_reopen), NULL); success = FALSE; } msg_set_context(NULL); if (success) { stats_counter_inc(self->stored_messages); step_sequence_number(&self->seq_num); log_msg_ack(msg, &path_options); log_msg_unref(msg); } else { log_queue_push_head(self->queue, msg, &path_options); } return success; }
static void log_threaded_dest_driver_do_insert(LogThrDestDriver *self) { LogMessage *msg; worker_insert_result_t result; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; while (!self->suspended && (msg = log_queue_pop_head(self->queue, &path_options)) != NULL) { msg_set_context(msg); log_msg_refcache_start_consumer(msg, &path_options); result = self->worker.insert(self, msg); switch (result) { case WORKER_INSERT_RESULT_DROP: log_threaded_dest_driver_message_drop(self, msg); _disconnect_and_suspend(self); break; case WORKER_INSERT_RESULT_ERROR: self->retries.counter++; if (self->retries.counter >= self->retries.max) { if (self->messages.retry_over) self->messages.retry_over(self, msg); log_threaded_dest_driver_message_drop(self, msg); } else { log_threaded_dest_driver_message_rewind(self, msg); _disconnect_and_suspend(self); } break; case WORKER_INSERT_RESULT_REWIND: log_threaded_dest_driver_message_rewind(self, msg); break; case WORKER_INSERT_RESULT_SUCCESS: log_threaded_dest_driver_message_accept(self, msg); break; default: break; } msg_set_context(NULL); log_msg_refcache_stop(); } }
static gpointer _threaded_consume(gpointer st) { LogQueue *q = (LogQueue *) st; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; gint loops = 0; gint msg_count = 0; /* just to make sure time is properly cached */ iv_init(); while (msg_count < MESSAGES_SUM) { gint slept = 0; msg = NULL; while((msg = log_queue_pop_head(q, &path_options)) == NULL) { struct timespec ns; /* sleep 1 msec */ ns.tv_sec = 0; ns.tv_nsec = 1000000; nanosleep(&ns, NULL); slept++; if (slept > 10000) { /* slept for more than 10 seconds */ fprintf(stderr, "The wait for messages took too much time, loops=%d, msg_count=%d\n", loops, msg_count); return GUINT_TO_POINTER(1); } } if ((loops % 10) == 0) { /* push the message back to the queue */ log_queue_push_head(q, msg, &path_options); } else { log_msg_ack(msg, &path_options, AT_PROCESSED); log_msg_unref(msg); msg_count++; } loops++; } iv_deinit(); return NULL; }
void send_some_messages(LogQueue *q, gint n, gboolean use_app_acks) { gint i; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; for (i = 0; i < n; i++) { log_queue_pop_head(q, &msg, &path_options, use_app_acks); log_msg_ack(msg, &path_options); log_msg_unref(msg); } }
void send_some_messages(LogQueue *q, gint n) { gint i; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; for (i = 0; i < n; i++) { msg = log_queue_pop_head(q, &path_options); log_msg_ack(msg, &path_options, AT_PROCESSED); log_msg_unref(msg); } }
static gboolean afmongodb_worker_insert (MongoDBDestDriver *self) { gboolean success; guint8 *oid; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; afmongodb_dd_connect(self, TRUE); g_mutex_lock(self->queue_mutex); log_queue_reset_parallel_push(self->queue); success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE); g_mutex_unlock(self->queue_mutex); if (!success) return TRUE; msg_set_context(msg); bson_reset (self->bson_sel); bson_reset (self->bson_upd); bson_reset (self->bson_set); oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num); bson_append_oid (self->bson_sel, "_id", oid); g_free (oid); bson_finish (self->bson_sel); value_pairs_foreach (self->vp, afmongodb_vp_foreach, msg, self->seq_num, self->bson_set); bson_finish (self->bson_set); bson_append_document (self->bson_upd, "$set", self->bson_set); bson_finish (self->bson_upd); if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT, self->bson_sel, self->bson_upd)) { msg_error ("Network error while inserting into MongoDB", evt_tag_int("time_reopen", self->time_reopen), NULL); success = FALSE; } msg_set_context(NULL); if (success) { stats_counter_inc(self->stored_messages); step_sequence_number(&self->seq_num); log_msg_ack(msg, &path_options); log_msg_unref(msg); } else { g_mutex_lock(self->queue_mutex); log_queue_push_head(self->queue, msg, &path_options); g_mutex_unlock(self->queue_mutex); } return success; }
/** * afsql_dd_insert_db: * * This function is running in the database thread * * Returns: FALSE to indicate that the connection should be closed and * this destination suspended for time_reopen() time. **/ static gboolean afsql_dd_insert_db(AFSqlDestDriver *self) { GString *table = NULL; GString *insert_command = NULL; LogMessage *msg; gboolean success; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; if (!afsql_dd_ensure_initialized_connection(self)) return FALSE; /* connection established, try to insert a message */ success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, self->flags & AFSQL_DDF_EXPLICIT_COMMITS); if (!success) return TRUE; msg_set_context(msg); table = afsql_dd_ensure_accessible_database_table(self, msg); if (!table) { success = FALSE; goto out; } if (afsql_dd_should_start_new_transaction(self) && !afsql_dd_begin_txn(self)) { success = FALSE; goto out; } insert_command = afsql_dd_build_insert_command(self, msg, table); success = afsql_dd_run_query(self, insert_command->str, FALSE, NULL); if (success && self->flush_lines_queued != -1) { self->flush_lines_queued++; if (afsql_dd_should_commit_transaction(self) && !afsql_dd_commit_txn(self)) { /* Assuming that in case of error, the queue is rewound by afsql_dd_commit_txn() */ g_string_free(insert_command, TRUE); msg_set_context(NULL); return FALSE; } } out: if (table != NULL) g_string_free(table, TRUE); if (insert_command != NULL) g_string_free(insert_command, TRUE); msg_set_context(NULL); if (success) { log_msg_ack(msg, &path_options); log_msg_unref(msg); step_sequence_number(&self->seq_num); self->failed_message_counter = 0; } else { if (self->failed_message_counter < self->num_retries - 1) { if (!afsql_dd_handle_insert_row_error_depending_on_connection_availability(self, msg, &path_options)) return FALSE; self->failed_message_counter++; } else { msg_error("Multiple failures while inserting this record into the database, message dropped", evt_tag_int("attempts", self->num_retries), NULL); stats_counter_inc(self->dropped_messages); log_msg_drop(msg, &path_options); self->failed_message_counter = 0; success = TRUE; } } return success; }
static gboolean perl_worker_eval(LogThrDestDriver *d) { PerlDestDriver *self = (PerlDestDriver *)d; gboolean success, vp_ok; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; PerlInterpreter *my_perl = self->perl; int count; HV *kvmap; gpointer args[3]; dSP; success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE); if (!success) return TRUE; msg_set_context(msg); ENTER; SAVETMPS; PUSHMARK(SP); kvmap = newHV(); args[0] = self->perl; args[1] = kvmap; args[2] = self; vp_ok = value_pairs_foreach(self->vp, perl_worker_vp_add_one, msg, self->seq_num, LTZ_SEND, &self->template_options, args); if (!vp_ok && (self->template_options.on_error & ON_ERROR_DROP_MESSAGE)) goto exit; XPUSHs(sv_2mortal(newRV_noinc((SV *)kvmap))); PUTBACK; count = call_pv(self->queue_func_name, G_EVAL | G_SCALAR); SPAGAIN; msg_set_context(NULL); if (SvTRUE(ERRSV)) { msg_error("Error while calling a Perl function", evt_tag_str("driver", self->super.super.super.id), evt_tag_str("script", self->filename), evt_tag_str("function", self->queue_func_name), evt_tag_str("error-message", SvPV_nolen(ERRSV)), NULL); (void) POPs; success = FALSE; } if (count != 1) { msg_error("Too many values returned by a Perl function", evt_tag_str("driver", self->super.super.super.id), evt_tag_str("script", self->filename), evt_tag_str("function", self->queue_func_name), evt_tag_int("returned-values", count), evt_tag_int("expected-values", 1), NULL); success = FALSE; } else { int r = POPi; success = (r != 0); } exit: PUTBACK; FREETMPS; LEAVE; if (success && vp_ok) { stats_counter_inc(self->super.stored_messages); step_sequence_number(&self->seq_num); log_msg_ack(msg, &path_options); log_msg_unref(msg); } else { stats_counter_inc(self->super.dropped_messages); step_sequence_number(&self->seq_num); log_msg_ack(msg, &path_options); log_msg_unref(msg); } return success; }
/** * afsql_dd_insert_db: * * This function is running in the database thread * * Returns: FALSE to indicate that the connection should be closed and * this destination suspended for time_reopen() time. **/ static gboolean afsql_dd_insert_db(AFSqlDestDriver *self) { GString *table, *query_string; LogMessage *msg; gboolean success; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; afsql_dd_connect(self); g_mutex_lock(self->db_thread_mutex); /* FIXME: this is a workaround because of the non-proper locking semantics * of the LogQueue. It might happen that the _queue() method sees 0 * elements in the queue, while the thread is still busy processing the * previous message. In that case arming the parallel push callback is * not needed and will cause assertions to fail. This is ugly and should * be fixed by properly defining the "blocking" semantics of the LogQueue * object w/o having to rely on user-code messing with parallel push * callbacks. */ log_queue_reset_parallel_push(self->queue); success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE); g_mutex_unlock(self->db_thread_mutex); if (!success) return TRUE; msg_set_context(msg); table = afsql_dd_validate_table(self, msg); if (!table) { /* If validate table is FALSE then close the connection and wait time_reopen time (next call) */ msg_error("Error checking table, disconnecting from database, trying again shortly", evt_tag_int("time_reopen", self->time_reopen), NULL); msg_set_context(NULL); g_string_free(table, TRUE); return afsql_dd_insert_fail_handler(self, msg, &path_options); } query_string = afsql_dd_construct_query(self, table, msg); if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self)) return FALSE; success = afsql_dd_run_query(self, query_string->str, FALSE, NULL); if (success && self->flush_lines_queued != -1) { self->flush_lines_queued++; if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE)) return FALSE; } g_string_free(table, TRUE); g_string_free(query_string, TRUE); msg_set_context(NULL); if (!success) return afsql_dd_insert_fail_handler(self, msg, &path_options); /* we only ACK if each INSERT is a separate transaction */ if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0) log_msg_ack(msg, &path_options); log_msg_unref(msg); step_sequence_number(&self->seq_num); self->failed_message_counter = 0; return TRUE; }
static gboolean redis_worker_insert(LogThrDestDriver *s) { RedisDriver *self = (RedisDriver *)s; gboolean success; LogMessage *msg; LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; redisReply *reply; const char *argv[5]; size_t argvlen[5]; int argc = 2; redis_dd_connect(self, TRUE); if (self->c->err) return FALSE; success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE); if (!success) return TRUE; msg_set_context(msg); log_template_format(self->key, msg, &self->template_options, LTZ_SEND, self->seq_num, NULL, self->key_str); if (self->param1) log_template_format(self->param1, msg, &self->template_options, LTZ_SEND, self->seq_num, NULL, self->param1_str); if (self->param2) log_template_format(self->param2, msg, &self->template_options, LTZ_SEND, self->seq_num, NULL, self->param2_str); argv[0] = self->command->str; argvlen[0] = self->command->len; argv[1] = self->key_str->str; argvlen[1] = self->key_str->len; if (self->param1) { argv[2] = self->param1_str->str; argvlen[2] = self->param1_str->len; argc++; } if (self->param2) { argv[3] = self->param2_str->str; argvlen[3] = self->param2_str->len; argc++; } reply = redisCommandArgv(self->c, argc, argv, argvlen); msg_debug("REDIS command sent", evt_tag_str("driver", self->super.super.super.id), evt_tag_str("command", self->command->str), evt_tag_str("key", self->key_str->str), evt_tag_str("param1", self->param1_str->str), evt_tag_str("param2", self->param2_str->str), NULL); success = TRUE; freeReplyObject(reply); msg_set_context(NULL); if (success) { stats_counter_inc(self->super.stored_messages); step_sequence_number(&self->seq_num); log_msg_ack(msg, &path_options); log_msg_unref(msg); } else { log_queue_push_head(self->super.queue, msg, &path_options); } return success; }