void *
fqd_worker_thread(void *arg)
{
struct incoming_message *m;
int tindex = *(int*) arg;
ck_fifo_spsc_t *q = &work_queues[tindex];
uint32_t *backlog = &work_queue_backlogs[tindex];
while (!worker_thread_shutdown) {
if (!CK_FIFO_SPSC_ISEMPTY(q)) {
if (ck_fifo_spsc_dequeue(q, &m)) {
if (m != NULL) {
ck_pr_dec_32(backlog);
fq_msg *copy = fq_msg_alloc_BLANK(m->msg->payload_len);
if (copy == NULL) {
continue;
}
memcpy(copy, m->msg, sizeof(fq_msg) + m->msg->payload_len);
/* reset the refcnt on the copy since the memcpy above will have overwritten it */
copy->refcnt = 1;
/* the copy will be freed as normal so eliminate cleanup_stack pointer */
copy->cleanup_stack = NULL;
/* we are done with the incoming message, deref */
fq_msg_deref(m->msg);
fqd_inject_message(m->client, copy);
fqd_remote_client_deref((remote_client *)m->client);
free(m);
}
}
} else {
usleep(1000);
}
}
/* drain the queue before exiting */
if (!CK_FIFO_SPSC_ISEMPTY(q)) {
ck_fifo_spsc_dequeue_lock(q);
if (ck_fifo_spsc_dequeue(q, &m)) {
if (m != NULL) {
ck_pr_dec_32(backlog);
fq_msg *copy = fq_msg_alloc_BLANK(m->msg->payload_len);
memcpy(copy, m->msg, sizeof(fq_msg) + m->msg->payload_len);
/* the copy will be freed as normal so eliminate cleanup_stack pointer */
copy->cleanup_stack = NULL;
/* we are done with the incoming message, drop it on it's cleanup stack */
fqd_inject_message(m->client, copy);
fq_msg_deref(m->msg);
free(m);
}
}
ck_fifo_spsc_dequeue_unlock(q);
}
usleep(10);
return NULL;
}
int main(int argc, char **argv) {
hrtime_t s, f;
uint64_t cnt = 0, icnt = 0, icnt_total = 0;
int rcvd = 0;
fq_client c;
fq_bind_req breq;
fq_msg *m;
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
memset(&breq, 0, sizeof(breq));
memcpy(breq.exchange.name, "maryland", 8);
breq.exchange.len = 8;
breq.peermode = 0;
breq.program = (char *)"prefix:\"test.prefix.\"";
fq_client_bind(c, &breq);
while(breq.out__route_id == 0) usleep(100);
printf("route set -> %u\n", breq.out__route_id);
if(breq.out__route_id == FQ_BIND_ILLEGAL) {
fprintf(stderr, "Failure to bind...\n");
exit(-1);
}
s = fq_gethrtime();
while(1) {
f = fq_gethrtime();
while(m = fq_client_receive(c)) {
icnt++;
icnt_total++;
rcvd++;
fq_msg_deref(m);
}
usleep(1000);
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
printf("total: %llu\n", (unsigned long long)icnt_total);
icnt = 0;
cnt = 0;
s = f;
}
}
(void) argc;
return 0;
}
static int
noit_fq_submit(iep_thread_driver_t *dr,
const char *payload, size_t payloadlen) {
int i;
struct fq_driver *driver = (struct fq_driver *)dr;
const char *routingkey = driver->routingkey;
fq_msg *msg;
if(*payload == 'M' ||
*payload == 'S' ||
*payload == 'C' ||
(*payload == 'B' && (payload[1] == '1' || payload[1] == '2'))) {
char uuid_str[32 * 2 + 1];
int account_id, check_id;
if(extract_uuid_from_jlog(payload, payloadlen,
&account_id, &check_id, uuid_str)) {
if(*routingkey) {
char *replace;
int newlen = strlen(driver->routingkey) + 1 + sizeof(uuid_str) + 2 * 32;
replace = alloca(newlen);
snprintf(replace, newlen, "%s.%x.%x.%d.%d%s", driver->routingkey,
account_id%16, (account_id/16)%16, account_id,
check_id, uuid_str);
routingkey = replace;
}
}
}
/* Setup our message */
msg = fq_msg_alloc(payload, payloadlen);
if(msg == NULL) {
driver->allocation_failures++;
return -1;
}
driver->msg_cnt++;
fq_msg_exchange(msg, driver->exchange, strlen(driver->exchange));
noitL(noit_debug, "route[%s] -> %s\n", driver->exchange, routingkey);
fq_msg_route(msg, routingkey, strlen(routingkey));
fq_msg_id(msg, NULL);
for(i=0; i<driver->nhosts; i++) {
if(fq_client_publish(driver->client[i], msg) == 1)
BUMPSTAT(i, publications);
else
BUMPSTAT(i, client_tx_drop);
}
fq_msg_deref(msg);
return 0;
}
int main(int argc, char **argv) {
hrtime_t s, f;
uint64_t cnt = 0, icnt = 0, icnt_total = 0;
int rcvd = 0;
fq_client c;
fq_msg *m;
char *fq_debug = getenv("FQ_DEBUG");
if(fq_debug) fq_debug_set_bits(atoi(fq_debug));
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(fq_client_hooks(c, &hooks)) {
fprintf(stderr, "Can't register hooks\n");
exit(-1);
}
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_hooks(c, &hooks);
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
s = fq_gethrtime();
while(1) {
f = fq_gethrtime();
while(m = fq_client_receive(c)) {
icnt++;
icnt_total++;
rcvd++;
fq_msg_deref(m);
}
usleep(1000);
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
printf("total: %llu\n", (unsigned long long)icnt_total);
icnt = 0;
cnt = 0;
s = f;
}
}
(void) argc;
return 0;
}
static void
fq_client_read_complete(void *closure, fq_msg *msg) {
fq_conn_s *conn_s = (fq_conn_s *)closure;
if(conn_s->message_hook && conn_s->message_hook(conn_s, msg)) {
fq_msg_deref(msg);
}
else {
ck_fifo_spsc_enqueue_lock(&conn_s->backq);
ck_fifo_spsc_entry_t *fifo_entry = ck_fifo_spsc_recycle(&conn_s->backq);
if (unlikely(fifo_entry == NULL)) {
fifo_entry = malloc(sizeof(ck_fifo_spsc_entry_t));
}
ck_fifo_spsc_enqueue(&conn_s->backq, fifo_entry, msg);
ck_fifo_spsc_enqueue_unlock(&conn_s->backq);
}
}
static void
http_req_clean(struct http_req *req) {
ck_ht_entry_t *cursor;
ck_ht_iterator_t iterator = CK_HT_ITERATOR_INITIALIZER;
while(ck_ht_next(&req->headers, &iterator, &cursor)) {
ck_ht_hash_t hv;
char *key = ck_ht_entry_key(cursor);
char *value = ck_ht_entry_value(cursor);
ck_ht_hash(&hv, &req->headers, key, strlen(key));
ck_ht_remove_spmc(&req->headers, hv, cursor);
free(key);
free(value);
}
ck_ht_iterator_init(&iterator);
while(ck_ht_next(&req->query_params, &iterator, &cursor)) {
ck_ht_hash_t hv;
char *key = ck_ht_entry_key(cursor);
char *value = ck_ht_entry_value(cursor);
ck_ht_hash(&hv, &req->headers, key, strlen(key));
ck_ht_remove_spmc(&req->headers, hv, cursor);
free(key);
free(value);
}
if(req->url) free(req->url);
/* req->qs isn't allocated */
if(req->status) free(req->status);
if(req->fldname) free(req->fldname);
if(req->error) free(req->error);
if(req->msg) fq_msg_deref(req->msg);
req->url = NULL;
req->qs = NULL;
req->status = NULL;
req->fldname = NULL;
req->error = NULL;
req->body_len = 0;
req->body_read = 0;
req->msg = NULL;
req->expect_continue = HTTP_EXPECT_NONE;
}
static void
fqd_data_driver(remote_client *parent) {
remote_data_client *me = parent->data;
fq_msg *inflight = NULL;
size_t inflight_sofar = 0;
buffered_msg_reader *ctx = NULL;
int had_msgs = 1, flags, needs_write = 1;
if(((flags = fcntl(me->fd, F_GETFL, 0)) == -1) ||
(fcntl(me->fd, F_SETFL, flags | O_NONBLOCK) == -1))
return;
ctx = fq_buffered_msg_reader_alloc(me->fd, (me->mode == FQ_PROTO_PEER_MODE));
while(1) {
uint32_t msgs_in = me->msgs_in, msgs_out = me->msgs_out;
int rv, timeout_ms = 1000;
struct pollfd pfd;
pfd.fd = me->fd;
pfd.events = POLLIN|POLLHUP;
if(needs_write) pfd.events |= POLLOUT;
pfd.revents = 0;
if(parent->heartbeat_ms && parent->heartbeat_ms < timeout_ms)
timeout_ms = parent->heartbeat_ms;
/* if we had msgs, but aren't waiting for write,
* then we set a very short timeout
*/
if(had_msgs && !needs_write) timeout_ms = 1;
rv = poll(&pfd, 1, timeout_ms);
if(rv < 0) break;
had_msgs = 0;
if(rv > 0 && (pfd.revents & POLLIN)) {
me->last_heartbeat = me->last_activity = fq_gethrtime();
if(fq_buffered_msg_read(ctx, fqd_dss_read_complete, parent) < 0) {
fq_debug(FQ_DEBUG_IO, "client read error\n");
break;
}
had_msgs = 1;
}
if(!needs_write || (rv > 0 && (pfd.revents & POLLOUT))) {
fq_msg *m;
needs_write = 0;
haveanother:
m = inflight ? inflight
: parent->queue ? fqd_queue_dequeue(parent->queue)
: NULL;
/* If we're in peer mode, we can just toss messages the remote has seen */
if(m && me->mode == FQ_PROTO_PEER_MODE &&
fq_find_in_hops(me->peer_id, m) >= 0) {
fq_msg_deref(m);
goto haveanother;
}
inflight = NULL;
while(m) {
int written;
size_t octets_out = 0;
had_msgs = 1;
written = fq_client_write_msg(me->fd, 1, m, inflight_sofar, &octets_out);
if(written > 0) inflight_sofar += written;
if(octets_out) me->octets_out += octets_out;
if(written > 0 || (written < 0 && errno == EAGAIN)) {
inflight = m;
needs_write = 1;
break;
}
else if(written < 0) {
fq_debug(FQ_DEBUG_IO, "client write error\n");
goto broken;
}
if(FQ_MESSAGE_DELIVER_ENABLED()) {
fq_dtrace_msg_t dmsg;
fq_dtrace_remote_client_t dpc;
fq_dtrace_remote_data_client_t dme;
DTRACE_PACK_MSG(&dmsg, m);
DTRACE_PACK_CLIENT(&dpc, parent);
DTRACE_PACK_DATA_CLIENT(&dme, me);
FQ_MESSAGE_DELIVER(&dpc, &dme, &dmsg);
}
fq_msg_deref(m);
me->msgs_out++;
inflight_sofar = 0;
m = parent->queue ? fqd_queue_dequeue(parent->queue) : NULL;
}
}
if(me->msgs_in != msgs_in || me->msgs_out != msgs_out)
fq_debug(FQ_DEBUG_MSG, "Round.... %d in, %d out\n", me->msgs_in, me->msgs_out);
}
broken:
if(inflight) {
/* We're screwed here... we might have delivered it. so just toss it */
fq_msg_deref(inflight);
}
if(ctx) fq_buffered_msg_reader_free(ctx);
parent->data = NULL;
fq_debug(FQ_DEBUG_IO, "data path from client ended: %s\n", parent->pretty);
}
static void
fq_data_worker_loop(fq_conn_s *conn_s) {
buffered_msg_reader *ctx = NULL;
ctx = fq_buffered_msg_reader_alloc(conn_s->data_fd, 1);
while(conn_s->cmd_fd >= 0 && conn_s->data_fd >= 0 && conn_s->stop == 0) {
int rv;
int wait_ms = 500, needs_write = 0, mask, write_rv;
if(conn_s->tosend) goto the_thick_of_it;
ck_fifo_spsc_dequeue_lock(&conn_s->q);
while(ck_fifo_spsc_dequeue(&conn_s->q, &conn_s->tosend) == true) {
conn_s->tosend_offset = 0;
ck_pr_dec_uint(&conn_s->qlen);
the_thick_of_it:
#ifdef DEBUG
fq_debug(FQ_DEBUG_MSG, "dequeue message to submit to server\n");
#endif
write_rv = fq_client_write_msg(conn_s->data_fd, conn_s->peermode,
conn_s->tosend, conn_s->tosend_offset, NULL);
if(write_rv > 0) {
conn_s->tosend_offset += write_rv;
wait_ms = 0;
break;
}
if(write_rv < 0) {
if(errno == EAGAIN) {
needs_write = 1;
break;
}
if(conn_s->errorlog) {
char errbuf[128];
snprintf(errbuf, sizeof(errbuf), "data write error: %s\n", strerror(errno));
conn_s->errorlog(conn_s, errbuf);
}
ck_fifo_spsc_dequeue_unlock(&conn_s->q);
goto finish;
}
fq_msg_deref(conn_s->tosend);
conn_s->tosend = NULL;
conn_s->tosend_offset = 0;
wait_ms = 0;
}
ck_fifo_spsc_dequeue_unlock(&conn_s->q);
rv = fq_client_wfrw_internal(conn_s->data_fd, 1, needs_write, wait_ms, &mask);
fq_debug(FQ_DEBUG_CONN, "fq_client_wfrw_internal(data:%d) -> %d\n", conn_s->data_fd, rv);
if(rv < 0) {
if(conn_s->errorlog) {
char errbuf[128];
snprintf(errbuf, sizeof(errbuf), "data read error: %s\n", strerror(errno));
conn_s->errorlog(conn_s, errbuf);
}
goto finish;
}
if(rv > 0 && (mask & POLLIN)) {
if(fq_buffered_msg_read(ctx, fq_client_read_complete, conn_s) < 0) {
if(conn_s->errorlog) conn_s->errorlog(conn_s, "data read: end-of-line\n");
goto finish;
}
}
}
finish:
fq_clear_message_cleanup_stack();
if(ctx) fq_buffered_msg_reader_free(ctx);
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "cmd_fd -> %d, stop -> %d\n", conn_s->cmd_fd, conn_s->stop);
#endif
}
int main(int argc, char **argv) {
hrtime_t s0, s, f, f0;
uint64_t cnt = 0, icnt = 0;
int psize = 0, i = 0, rcvd = 0;
fq_client c;
fq_bind_req breq;
fq_msg *m;
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
memset(&breq, 0, sizeof(breq));
memcpy(breq.exchange.name, "maryland", 8);
breq.exchange.len = 8;
breq.peermode = 0;
breq.program = (char *)"prefix:\"test.prefix.\"";
fq_client_bind(c, &breq);
while(breq.out__route_id == 0) usleep(100);
printf("route set -> %u\n", breq.out__route_id);
if(breq.out__route_id == FQ_BIND_ILLEGAL) {
fprintf(stderr, "Failure to bind...\n");
exit(-1);
}
if(argc > 5) {
psize = atoi(argv[5]);
}
printf("payload size -> %d\n", psize);
if(argc > 6) {
send_count = atoi(argv[6]);
}
printf("message count -> %d\n", send_count);
s0 = s = fq_gethrtime();
while(i < send_count || fq_client_data_backlog(c) > 0) {
if(i < send_count) {
m = fq_msg_alloc_BLANK(psize);
memset(m->payload, 0, psize);
fq_msg_exchange(m, "maryland", 8);
fq_msg_route(m, "test.prefix.foo", 15);
fq_msg_id(m, NULL);
fq_client_publish(c, m);
cnt++;
i++;
fq_msg_free(m);
}
else usleep(100);
f = fq_gethrtime();
while(m = fq_client_receive(c)) {
icnt++;
rcvd++;
fq_msg_deref(m);
}
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
icnt = 0;
cnt = 0;
s = f;
}
}
f0 = fq_gethrtime();
print_rate(c, s0, f0, i, 0);
do {
icnt=0;
while(m = fq_client_receive(c)) {
icnt++;
rcvd++;
fq_msg_deref(m);
}
} while(rcvd < send_count);
f0 = fq_gethrtime();
print_rate(c, s0, f0, 0, rcvd);
printf("Total received during test: %d\n", rcvd);
(void) argc;
return 0;
}
void fq_buffered_msg_reader_free(buffered_msg_reader *f) {
assert(f->msg->refcnt == 1);
fq_msg_deref(f->msg);
if(f->copy) fq_msg_deref(f->copy);
free(f);
}
static int
noit_fq_submit(iep_thread_driver_t *dr,
const char *payload, size_t payloadlen) {
int i;
struct fq_driver *driver = (struct fq_driver *)dr;
const char *routingkey = driver->routingkey;
mtev_hash_table *filtered_metrics;
char uuid_formatted_str[UUID_STR_LEN+1];
bool is_bundle = false, is_metric = false, send = false;
fq_msg *msg;
char *metric = NULL;
if(*payload == 'M' ||
*payload == 'S' ||
*payload == 'C' ||
(*payload == 'H' && payload[1] == '1') ||
(*payload == 'F' && payload[1] == '1') ||
(*payload == 'B' && (payload[1] == '1' || payload[1] == '2'))) {
char uuid_str[32 * 2 + 1];
int account_id, check_id;
if (*payload == 'B') is_bundle = true;
else if ((*payload == 'H') || (*payload == 'M')) {
is_metric = true;
}
if(extract_uuid_from_jlog(payload, payloadlen,
&account_id, &check_id, uuid_str,
uuid_formatted_str, &metric)) {
if(*routingkey) {
char *replace;
int newlen = strlen(driver->routingkey) + 1 + sizeof(uuid_str) + 2 * 32;
replace = alloca(newlen);
snprintf(replace, newlen, "%s.%x.%x.%d.%d%s", driver->routingkey,
account_id%16, (account_id/16)%16, account_id,
check_id, uuid_str);
routingkey = replace;
}
}
}
/* Let through any messages that aren't metrics or bundles */
if (!is_bundle && !is_metric) {
send = true;
}
/* Setup our message */
msg = fq_msg_alloc(payload, payloadlen);
if(msg == NULL) {
driver->allocation_failures++;
if (metric) free(metric);
return -1;
}
driver->msg_cnt++;
fq_msg_exchange(msg, driver->exchange, strlen(driver->exchange));
mtevL(mtev_debug, "route[%s] -> %s\n", driver->exchange, routingkey);
fq_msg_route(msg, routingkey, strlen(routingkey));
fq_msg_id(msg, NULL);
if (global_fq_ctx.round_robin) {
int checked = 0, good = 0;
time_t cur_time;
while (1) {
if (!global_fq_ctx.down_host[global_fq_ctx.round_robin_target]) {
good = 1;
break;
}
cur_time = time(NULL);
if (cur_time - global_fq_ctx.last_error[global_fq_ctx.round_robin_target] >= 10) {
global_fq_ctx.down_host[global_fq_ctx.round_robin_target] = false;
good = 1;
break;
}
global_fq_ctx.round_robin_target = (global_fq_ctx.round_robin_target+1) % driver->nhosts;
checked++;
if (checked == driver->nhosts) {
/* This means everybody is down.... just try to send to whatever fq
we're pointing at */
break;
}
}
if (good) {
if(fq_client_publish(driver->client[global_fq_ctx.round_robin_target], msg) == 1) {
BUMPSTAT(global_fq_ctx.round_robin_target, publications);
}
else {
BUMPSTAT(global_fq_ctx.round_robin_target, client_tx_drop);
}
}
/* Go ahead and try to publish to the hosts that are down, just in
case they've come back up. This should help minimize lost messages */
for (i=0; i<driver->nhosts; i++) {
if (global_fq_ctx.down_host[i]) {
if(fq_client_publish(driver->client[i], msg) == 1) {
BUMPSTAT(i, publications);
}
else {
BUMPSTAT(i, client_tx_drop);
}
}
}
global_fq_ctx.round_robin_target = (global_fq_ctx.round_robin_target+1) % driver->nhosts;
}
else {
for(i=0; i<driver->nhosts; i++) {
if(fq_client_publish(driver->client[i], msg) == 1) {
BUMPSTAT(i, publications);
}
else {
BUMPSTAT(i, client_tx_drop);
}
}
}
fq_msg_deref(msg);
if (!send) {
if(mtev_hash_retrieve(&filtered_checks_hash, uuid_formatted_str, strlen(uuid_formatted_str), (void**)&filtered_metrics)) {
if (is_bundle || (is_metric && mtev_hash_size(filtered_metrics) == 0)) {
send = true;
}
else if (is_metric) {
void *tmp;
if (mtev_hash_retrieve(filtered_metrics, metric, strlen(metric), &tmp)) {
send = true;
}
}
}
}
if (global_fq_ctx.filtered_exchange[0] && send) {
fq_msg *msg2;
msg2 = fq_msg_alloc(payload, payloadlen);
fq_msg_exchange(msg2, driver->filtered_exchange, strlen(driver->filtered_exchange));
mtevL(mtev_debug, "route[%s] -> %s\n", driver->filtered_exchange, routingkey);
fq_msg_route(msg2, routingkey, strlen(routingkey));
fq_msg_id(msg2, NULL);
for(i=0; i<driver->nhosts; i++) {
if(fq_client_publish(driver->client[i], msg2) == 1) {
BUMPSTAT(i, publications);
}
else {
BUMPSTAT(i, client_tx_drop);
}
}
fq_msg_deref(msg2);
}
if (metric) free(metric);
return 0;
}