int main(int argc, char **argv) {
hrtime_t s0, s, f, f0;
uint64_t cnt = 0, icnt = 0;
int psize = 0, i = 0;
fq_client c;
fq_msg *m;
char *fq_debug = getenv("FQ_DEBUG");
if(fq_debug) fq_debug_set_bits(atoi(fq_debug));
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
if(argc > 5) {
psize = atoi(argv[5]);
}
printf("payload size -> %d\n", psize);
if(argc > 6) {
send_count = atoi(argv[6]);
}
printf("message count -> %d\n", send_count);
s0 = s = fq_gethrtime();
while(i < send_count || fq_client_data_backlog(c) > 0) {
if(i < send_count) {
m = fq_msg_alloc_BLANK(psize);
memset(m->payload, 0, psize);
fq_msg_exchange(m, "maryland", 8);
fq_msg_route(m, "test.prefix.boo", 15);
fq_msg_id(m, NULL);
fq_client_publish(c, m);
cnt++;
i++;
fq_msg_free(m);
}
else usleep(100);
f = fq_gethrtime();
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
icnt = 0;
cnt = 0;
s = f;
}
}
f0 = fq_gethrtime();
print_rate(c, s0, f0, i, 0);
(void) argc;
return 0;
}
int main(int argc, char **argv) {
hrtime_t s, f;
uint64_t cnt = 0, icnt = 0, icnt_total = 0;
int rcvd = 0;
fq_client c;
fq_bind_req breq;
fq_msg *m;
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
memset(&breq, 0, sizeof(breq));
memcpy(breq.exchange.name, "maryland", 8);
breq.exchange.len = 8;
breq.peermode = 0;
breq.program = (char *)"prefix:\"test.prefix.\"";
fq_client_bind(c, &breq);
while(breq.out__route_id == 0) usleep(100);
printf("route set -> %u\n", breq.out__route_id);
if(breq.out__route_id == FQ_BIND_ILLEGAL) {
fprintf(stderr, "Failure to bind...\n");
exit(-1);
}
s = fq_gethrtime();
while(1) {
f = fq_gethrtime();
while(m = fq_client_receive(c)) {
icnt++;
icnt_total++;
rcvd++;
fq_msg_deref(m);
}
usleep(1000);
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
printf("total: %llu\n", (unsigned long long)icnt_total);
icnt = 0;
cnt = 0;
s = f;
}
}
(void) argc;
return 0;
}
static int
fq_socket_connect(fq_conn_s *conn_s) {
int fd, rv, on = 1;
/* re-resolve if we've not done so quite recently */
if(conn_s->last_resolve == 0 ||
(fq_gethrtime() - conn_s->last_resolve) > MAX_RESOLVE_CACHE) {
if(fq_resolve_endpoint(conn_s) < 0) {
return -1;
}
}
fd = socket(AF_INET, SOCK_STREAM, 0);
if(fd == -1) return -1;
rv = connect(fd, (struct sockaddr *)&conn_s->remote,
sizeof(conn_s->remote));
if(rv == -1) {
snprintf(conn_s->error, sizeof(conn_s->error),
"socket: %s", strerror(errno));
CONNERR_S(conn_s);
close(fd);
return -1;
}
/* If this fails, we ignore it */
(void)setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));
return fd;
}
int main(int argc, char **argv) {
hrtime_t s, f;
uint64_t cnt = 0, icnt = 0, icnt_total = 0;
int rcvd = 0;
fq_client c;
fq_msg *m;
char *fq_debug = getenv("FQ_DEBUG");
if(fq_debug) fq_debug_set_bits(atoi(fq_debug));
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(fq_client_hooks(c, &hooks)) {
fprintf(stderr, "Can't register hooks\n");
exit(-1);
}
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_hooks(c, &hooks);
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
s = fq_gethrtime();
while(1) {
f = fq_gethrtime();
while(m = fq_client_receive(c)) {
icnt++;
icnt_total++;
rcvd++;
fq_msg_deref(m);
}
usleep(1000);
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
printf("total: %llu\n", (unsigned long long)icnt_total);
icnt = 0;
cnt = 0;
s = f;
}
}
(void) argc;
return 0;
}
static int
fq_resolve_endpoint(fq_conn_s *conn_s) {
conn_s->remote.sin_family = AF_INET;
conn_s->remote.sin_port = htons(conn_s->port);
if(inet_pton(AF_INET, conn_s->host, &conn_s->remote.sin_addr) != 1) {
#ifdef HAVE_GETHOSTBYNAME_R
struct hostent hostbuf, *hp;
struct in_addr **addr_list;
int buflen = 1024, herr;
char *buf;
if((buf = malloc(buflen)) == NULL) {
CONNERR(conn_s, "out of memory");
return -1;
}
while((hp = gethostbyname_r(conn_s->host, &hostbuf, buf, buflen, &herr)) == NULL &&
errno == ERANGE) {
buflen *= 2;
if((buf = realloc(buf, buflen)) == NULL) {
CONNERR(conn_s, "out of memory");
free(buf);
return -1;
}
}
if(!hp) {
CONNERR(conn_s, "host lookup failed");
free(buf);
return -1;
}
addr_list = (struct in_addr **)hp->h_addr_list;
if(*addr_list == 0) {
CONNERR(conn_s, "no address for host");
free(buf);
return -1;
}
memcpy(&conn_s->remote.sin_addr, *addr_list, sizeof(struct in_addr));
free(buf);
#else
struct hostent *hp;
struct in_addr **addr_list;
hp = gethostbyname(conn_s->host);
if(!hp) {
CONNERR(conn_s, "host lookup failed");
return -1;
}
addr_list = (struct in_addr **)hp->h_addr_list;
if(*addr_list == 0) {
CONNERR(conn_s, "no address for host");
return -1;
}
memcpy(&conn_s->remote.sin_addr, *addr_list, sizeof(struct in_addr));
#endif
}
conn_s->last_resolve = fq_gethrtime();
return 0;
}
fq_msg *
fq_msg_alloc_BLANK(size_t s) {
fq_msg *m;
m = calloc(offsetof(fq_msg, payload) + ((s | (MSG_ALIGN-1))+1), 1);
if(!m) return NULL;
m->payload_len = s;
#ifdef DEBUG
fq_debug(FQ_DEBUG_MSG, "msg(%p) -> alloc\n", (void *)m);
#endif
m->arrival_time = fq_gethrtime();
m->refcnt = 1;
return m;
}
fq_msg *
fq_msg_alloc(const void *data, size_t s) {
fq_msg *m;
m = malloc(offsetof(fq_msg, payload) + ((s | (MSG_ALIGN-1))+1));
if(!m) return NULL;
memset(m, 0, offsetof(fq_msg, payload));
m->payload_len = s;
if(s) memcpy(m->payload, data, s);
#ifdef DEBUG
fq_debug(FQ_DEBUG_MSG, "msg(%p) -> alloc\n", (void *)m);
#endif
m->arrival_time = fq_gethrtime();
m->refcnt = 1;
return m;
}
int
fq_debug_fl(const char *file, int line, fq_debug_bits_t b, const char *fmt, ...) {
int rv;
va_list argp;
static hrtime_t epoch = 0;
hrtime_t now;
char fmtstring[1024];
uint64_t p = (uint64_t)pthread_self();
uint32_t ps = p & 0xffffffff;
(void)b;
now = fq_gethrtime();
if(!epoch) epoch = now;
snprintf(fmtstring, sizeof(fmtstring), "[%llu] [%08x] %s",
(now-epoch)/1000, ps, fmt);
va_start(argp, fmt);
rv = vfprintf(stderr, fmtstring, argp);
va_end(argp);
(void)file;
(void)line;
return rv;
}
static void
fqd_data_driver(remote_client *parent) {
remote_data_client *me = parent->data;
fq_msg *inflight = NULL;
size_t inflight_sofar = 0;
buffered_msg_reader *ctx = NULL;
int had_msgs = 1, flags, needs_write = 1;
if(((flags = fcntl(me->fd, F_GETFL, 0)) == -1) ||
(fcntl(me->fd, F_SETFL, flags | O_NONBLOCK) == -1))
return;
ctx = fq_buffered_msg_reader_alloc(me->fd, (me->mode == FQ_PROTO_PEER_MODE));
while(1) {
uint32_t msgs_in = me->msgs_in, msgs_out = me->msgs_out;
int rv, timeout_ms = 1000;
struct pollfd pfd;
pfd.fd = me->fd;
pfd.events = POLLIN|POLLHUP;
if(needs_write) pfd.events |= POLLOUT;
pfd.revents = 0;
if(parent->heartbeat_ms && parent->heartbeat_ms < timeout_ms)
timeout_ms = parent->heartbeat_ms;
/* if we had msgs, but aren't waiting for write,
* then we set a very short timeout
*/
if(had_msgs && !needs_write) timeout_ms = 1;
rv = poll(&pfd, 1, timeout_ms);
if(rv < 0) break;
had_msgs = 0;
if(rv > 0 && (pfd.revents & POLLIN)) {
me->last_heartbeat = me->last_activity = fq_gethrtime();
if(fq_buffered_msg_read(ctx, fqd_dss_read_complete, parent) < 0) {
fq_debug(FQ_DEBUG_IO, "client read error\n");
break;
}
had_msgs = 1;
}
if(!needs_write || (rv > 0 && (pfd.revents & POLLOUT))) {
fq_msg *m;
needs_write = 0;
haveanother:
m = inflight ? inflight
: parent->queue ? fqd_queue_dequeue(parent->queue)
: NULL;
/* If we're in peer mode, we can just toss messages the remote has seen */
if(m && me->mode == FQ_PROTO_PEER_MODE &&
fq_find_in_hops(me->peer_id, m) >= 0) {
fq_msg_deref(m);
goto haveanother;
}
inflight = NULL;
while(m) {
int written;
size_t octets_out = 0;
had_msgs = 1;
written = fq_client_write_msg(me->fd, 1, m, inflight_sofar, &octets_out);
if(written > 0) inflight_sofar += written;
if(octets_out) me->octets_out += octets_out;
if(written > 0 || (written < 0 && errno == EAGAIN)) {
inflight = m;
needs_write = 1;
break;
}
else if(written < 0) {
fq_debug(FQ_DEBUG_IO, "client write error\n");
goto broken;
}
if(FQ_MESSAGE_DELIVER_ENABLED()) {
fq_dtrace_msg_t dmsg;
fq_dtrace_remote_client_t dpc;
fq_dtrace_remote_data_client_t dme;
DTRACE_PACK_MSG(&dmsg, m);
DTRACE_PACK_CLIENT(&dpc, parent);
DTRACE_PACK_DATA_CLIENT(&dme, me);
FQ_MESSAGE_DELIVER(&dpc, &dme, &dmsg);
}
fq_msg_deref(m);
me->msgs_out++;
inflight_sofar = 0;
m = parent->queue ? fqd_queue_dequeue(parent->queue) : NULL;
}
}
if(me->msgs_in != msgs_in || me->msgs_out != msgs_out)
fq_debug(FQ_DEBUG_MSG, "Round.... %d in, %d out\n", me->msgs_in, me->msgs_out);
}
broken:
if(inflight) {
/* We're screwed here... we might have delivered it. so just toss it */
fq_msg_deref(inflight);
}
if(ctx) fq_buffered_msg_reader_free(ctx);
parent->data = NULL;
fq_debug(FQ_DEBUG_IO, "data path from client ended: %s\n", parent->pretty);
}
static void *
fq_conn_worker(void *u) {
int backoff = 0, i;
bool zero;
fq_conn_s *conn_s = (fq_conn_s *)u;
cmd_instr *last_entries[MAX_PENDING] = { NULL };
int last_entry_idx = 0;
int next_entry_idx = 0;
#define SAVE_ENTRY(e) do { \
if(last_entries[next_entry_idx] != NULL) { \
CONNERR(conn_s, "exceed max cmd pipeline"); \
goto restart; \
} \
last_entries[next_entry_idx++] = e; \
next_entry_idx = next_entry_idx % MAX_PENDING; \
} while(0)
#define last_entry last_entries[last_entry_idx]
#define PROCESS_ENTRY(e, should_free) do { \
fq_assert(last_entries[last_entry_idx] == e); \
if(should_free) free(last_entries[last_entry_idx]); \
last_entries[last_entry_idx++] = NULL; \
last_entry_idx = last_entry_idx % MAX_PENDING; \
} while(0)
cmd_instr *entry;
ck_pr_inc_uint(&conn_s->thrcnt);
while(conn_s->stop == 0) {
int wait_ms = 50;
if(fq_client_connect_internal(conn_s) == 0) {
backoff = 0; /* we're good, restart our backoff */
}
while(conn_s->data_ready && conn_s->stop == 0) {
hrtime_t t;
unsigned long long hb_us;
struct timeval tv;
int rv;
ck_fifo_spsc_dequeue_lock(&conn_s->cmdq);
while(ck_fifo_spsc_dequeue(&conn_s->cmdq, &entry) == true) {
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "client acting on user req 0x%04x\n", entry->cmd);
#endif
switch(entry->cmd) {
case FQ_PROTO_STATUSREQ:
fq_debug(FQ_DEBUG_CONN, "sending status request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd)) {
free(entry);
CONNERR(conn_s, "write failed (statusreq)");
goto restart;
}
SAVE_ENTRY(entry);
break;
case FQ_PROTO_HBREQ:
fq_debug(FQ_DEBUG_CONN, "sending heartbeat request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, entry->data.heartbeat.ms)) {
free(entry);
CONNERR(conn_s, "write failed (hbreq)");
goto restart;
}
conn_s->cmd_hb_ms = entry->data.heartbeat.ms;
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_RCVTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_SNDTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
conn_s->cmd_hb_last = fq_gethrtime();
free(entry);
break;
case FQ_PROTO_BINDREQ:
{
unsigned short flags = entry->data.bind->flags;
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, flags) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.bind->exchange.len,
entry->data.bind->exchange.name) < 0 ||
fq_write_short_cmd(conn_s->cmd_fd,
strlen(entry->data.bind->program),
entry->data.bind->program) < 0) {
CONNERR(conn_s, "write failed (bindreq)");
goto restart;
}
SAVE_ENTRY(entry);
}
break;
case FQ_PROTO_UNBINDREQ:
{
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint32(conn_s->cmd_fd, entry->data.unbind->route_id) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.unbind->exchange.len,
entry->data.unbind->exchange.name) < 0) {
CONNERR(conn_s, "write failed (unbindreq)");
goto restart;
}
SAVE_ENTRY(entry);
}
break;
default:
CONNERR(conn_s, "unknown user-side cmd");
free(entry);
}
}
ck_fifo_spsc_dequeue_unlock(&conn_s->cmdq);
if(conn_s->cmd_hb_needed) {
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "-> heartbeat\n");
#endif
if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_HB)) {
CONNERR(conn_s, "write failed (hb)");
break;
}
conn_s->cmd_hb_needed = 0;
}
rv = fq_client_wfrw_internal(conn_s->cmd_fd, 1, 0, wait_ms, NULL);
if(rv == 0) {
wait_ms = (wait_ms >> 2) + wait_ms;
if(conn_s->cmd_hb_ms && wait_ms > conn_s->cmd_hb_ms)
wait_ms = conn_s->cmd_hb_ms;
if(wait_ms > 1000) wait_ms = 1000;
}
else wait_ms = 50;
fq_debug(FQ_DEBUG_CONN, "fq_client_wfrw_internal(cmd:%d) -> %d\n", conn_s->cmd_fd, rv);
t = fq_gethrtime();
hb_us = (unsigned long long)conn_s->cmd_hb_ms * 3 * 1000000ULL;
if(conn_s->cmd_hb_last && hb_us &&
conn_s->cmd_hb_last < (unsigned int) (t - hb_us)) {
char errbuf[256];
snprintf(errbuf, sizeof(errbuf), "heartbeat failed [%llu - %llu = %llu]",
(unsigned long long)t, (unsigned long long)conn_s->cmd_hb_last,
(unsigned long long)(t - conn_s->cmd_hb_last));
CONNERR(conn_s, errbuf);
break;
}
if(rv < 0) {
CONNERR(conn_s, strerror(errno));
break;
}
if(rv > 0) {
bool handled = false;
uint16_t hb;
if(fq_read_uint16(conn_s->cmd_fd, &hb)) break;
switch(hb) {
case FQ_PROTO_HB:
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "<- heartbeat\n");
#endif
conn_s->cmd_hb_last = fq_gethrtime();
conn_s->cmd_hb_needed = 1;
break;
case FQ_PROTO_STATUS:
if(!last_entry || last_entry->cmd != FQ_PROTO_STATUSREQ) {
CONNERR(conn_s, "protocol violation (status unexpected)");
goto restart;
}
if(fq_read_status(conn_s->cmd_fd,
last_entry->data.status.callback,
last_entry->data.status.closure))
goto restart;
PROCESS_ENTRY(last_entry, 1);
break;
case FQ_PROTO_BIND:
if(!last_entry || last_entry->cmd != FQ_PROTO_BINDREQ) {
CONNERR(conn_s, "protocol violation (bind unexpected)");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.bind->out__route_id)) {
if(conn_s->bind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
}
else conn_s->bind_hook((fq_client)conn_s, last_entry->data.bind);
}
CONNERR(conn_s, "read failed (bind)");
goto restart;
}
if(conn_s->bind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
}
else conn_s->bind_hook((fq_client)conn_s, last_entry->data.bind);
}
if(!handled) PROCESS_ENTRY(last_entry, 1);
break;
case FQ_PROTO_UNBIND:
if(!last_entry || last_entry->cmd != FQ_PROTO_UNBINDREQ) {
CONNERR(conn_s, "protocol violation (unbind unexpected)");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.unbind->out__success)) {
if(conn_s->unbind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
}
conn_s->unbind_hook((fq_client)conn_s, last_entry->data.unbind);
}
CONNERR(conn_s, "read failed (unbind)");
goto restart;
}
if(conn_s->unbind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
last_entry = NULL;
}
conn_s->unbind_hook((fq_client)conn_s, last_entry->data.unbind);
}
if(!handled) PROCESS_ENTRY(last_entry,1);
break;
default:
CONNERR(conn_s, "protocol violation");
goto restart;
break;
}
}
}
int main(int argc, char **argv) {
hrtime_t s0, s, f, f0;
uint64_t cnt = 0, icnt = 0;
int psize = 0, i = 0, rcvd = 0;
fq_client c;
fq_bind_req breq;
fq_msg *m;
signal(SIGPIPE, SIG_IGN);
fq_client_init(&c, 0, logger);
if(argc < 5) {
fprintf(stderr, "%s <host> <port> <user> <pass> [size [count]]\n",
argv[0]);
exit(-1);
}
fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]);
fq_client_heartbeat(c, 1000);
fq_client_set_backlog(c, 10000, 100);
fq_client_connect(c);
memset(&breq, 0, sizeof(breq));
memcpy(breq.exchange.name, "maryland", 8);
breq.exchange.len = 8;
breq.peermode = 0;
breq.program = (char *)"prefix:\"test.prefix.\"";
fq_client_bind(c, &breq);
while(breq.out__route_id == 0) usleep(100);
printf("route set -> %u\n", breq.out__route_id);
if(breq.out__route_id == FQ_BIND_ILLEGAL) {
fprintf(stderr, "Failure to bind...\n");
exit(-1);
}
if(argc > 5) {
psize = atoi(argv[5]);
}
printf("payload size -> %d\n", psize);
if(argc > 6) {
send_count = atoi(argv[6]);
}
printf("message count -> %d\n", send_count);
s0 = s = fq_gethrtime();
while(i < send_count || fq_client_data_backlog(c) > 0) {
if(i < send_count) {
m = fq_msg_alloc_BLANK(psize);
memset(m->payload, 0, psize);
fq_msg_exchange(m, "maryland", 8);
fq_msg_route(m, "test.prefix.foo", 15);
fq_msg_id(m, NULL);
fq_client_publish(c, m);
cnt++;
i++;
fq_msg_free(m);
}
else usleep(100);
f = fq_gethrtime();
while(m = fq_client_receive(c)) {
icnt++;
rcvd++;
fq_msg_deref(m);
}
if(f-s > 1000000000) {
print_rate(c, s, f, cnt, icnt);
icnt = 0;
cnt = 0;
s = f;
}
}
f0 = fq_gethrtime();
print_rate(c, s0, f0, i, 0);
do {
icnt=0;
while(m = fq_client_receive(c)) {
icnt++;
rcvd++;
fq_msg_deref(m);
}
} while(rcvd < send_count);
f0 = fq_gethrtime();
print_rate(c, s0, f0, 0, rcvd);
printf("Total received during test: %d\n", rcvd);
(void) argc;
return 0;
}
/*
* return 0: keep going (to write path)
* return -1: busted
*
* Read into one of N buffers so the processing thread
* can do the work separate from the read
*/
int
fq_buffered_msg_read(buffered_msg_reader *f,
void (*f_msg_handler)(void *, fq_msg *),
void *closure) {
int rv;
static char scratch_buf[IN_READ_BUFFER_SIZE];
while(f->into_body < f->msg->payload_len) {
fq_assert(f->copy);
/* we need to be reading a largish payload */
if(f->into_body >= MAX_MESSAGE_SIZE) {
/* read into a scratch buffer */
size_t readsize = f->copy->payload_len - f->into_body;
if(readsize > sizeof(scratch_buf)) readsize = sizeof(scratch_buf);
while((rv = read(f->fd, scratch_buf, readsize)) == -1 && errno == EINTR);
}
else {
while((rv = read(f->fd, f->copy->payload + f->into_body,
CAPPED(f->copy->payload_len) - f->into_body)) == -1 && errno == EINTR);
}
if(rv < 0 && errno == EAGAIN) return 0;
if(rv <= 0) {
fq_debug(FQ_DEBUG_IO, "read error: %s\n", rv < 0 ? strerror(errno) : "end-of-line");
return -1;
}
fq_debug(FQ_DEBUG_MSG, "%p <-- %d bytes for payload\n", (void *)f, rv);
f->into_body += rv;
if(f->into_body == f->copy->payload_len) {
f->into_body = 0;
goto message_done;
}
}
while((rv = read(f->fd, f->scratch+f->nread, sizeof(f->scratch)-f->nread)) == -1 &&
errno == EINTR);
fq_debug(FQ_DEBUG_IO, "%p <-- %d bytes @ %d (%d)\n", (void *)f, rv, (int)f->nread,
(int)f->nread + ((rv > 0) ? rv : 0));
if(rv == -1 && errno == EAGAIN) return 0;
if(rv <= 0) return -1;
f->nread += rv;
while(f->nread>0) {
uint32_t body_available;
int body_start;
body_start = parse_message_headers(f->peermode,
f->scratch+f->off, f->nread-f->off,
f->msg);
f->into_body = 0;
fq_debug(FQ_DEBUG_MSG, "%d = parse(+%d, %d) -> %d\n",
body_start, f->off, (int)f->nread-f->off,
body_start ? (int)f->msg->payload_len : 0);
if(body_start < 0) return -1;
if(!body_start) {
fq_debug(FQ_DEBUG_MSG, "incomplete message header...\n");
memmove(f->scratch, f->scratch + f->off, f->nread - f->off);
f->nread -= f->off;
f->off = 0;
return 0;
}
free_message_stack *tls_free_message_stack = NULL;
int msg_stack_idx = msg_free_stack_select(f->msg->payload_len);
if(msg_stack_idx >= 0) {
if(tls_free_message_handle == NULL)
tls_free_message_handle = free_message_handle_acquire();
if(tls_free_message_handle->stacks[msg_stack_idx] == NULL) {
/* lazy create/init the cleanup stack */
tls_free_message_handle->stacks[msg_stack_idx] = malloc(sizeof(free_message_stack));
fq_init_free_message_stack(tls_free_message_handle->stacks[msg_stack_idx],
FREE_MSG_LIST_SIZE/(1 << (msg_stack_idx + MSG_FREE_BASE)),
(1 << (msg_stack_idx + MSG_FREE_BASE)));
}
tls_free_message_stack = tls_free_message_handle->stacks[msg_stack_idx];
}
if(tls_free_message_stack) {
/* We have a message... or the formal beginnings of one */
f->copy = fq_pop_free_message_stack(tls_free_message_stack);
if (f->copy == NULL) {
/* ran out of entries in free list */
f->copy = fq_msg_alloc_BLANK(tls_free_message_stack->alloc_size);
if (f->copy == NULL) {
/* this is bad, we can't alloc */
fq_debug(FQ_DEBUG_MSG, "unable to malloc, OOM?\n");
return -1;
}
}
/* always 1 as this msg only lives until it's copied by a worker thread */
memcpy(f->copy, f->msg, sizeof(fq_msg));
f->copy->refcnt = 1;
f->copy->free_fn = fq_free_msg_fn;
} else {
f->copy = fq_msg_alloc_BLANK(CAPPED(f->msg->payload_len));
if (f->copy == NULL) {
/* this is bad, we can't alloc */
fq_debug(FQ_DEBUG_MSG, "unable to malloc, OOM?\n");
return -1;
}
memcpy(f->copy, f->msg, sizeof(fq_msg));
f->copy->refcnt = 1;
f->copy->free_fn = NULL;
}
/* assign the cleanup stack for this message */
f->copy->cleanup_handle = tls_free_message_stack ? tls_free_message_handle : NULL;
memset(&f->copy->cleanup_stack_entry, 0, sizeof(ck_stack_entry_t));
f->off += body_start;
body_available = f->nread - f->off;
if(f->copy->payload_len < body_available) body_available = f->copy->payload_len;
memcpy(f->copy->payload, f->scratch+f->off, CAPPED(body_available));
if(body_available == f->copy->payload_len) {
f->off += body_available;
message_done:
f->copy->refcnt = 1;
f->copy->payload_len = CAPPED(f->copy->payload_len);
fq_debug(FQ_DEBUG_MSG, "message read... injecting\n");
f->copy->arrival_time = fq_gethrtime();
f_msg_handler(closure, f->copy);
f->copy = NULL;
memset(f->msg, 0, sizeof(fq_msg));
/* It is still allocated and we are the sole owner, refcnt must be 1 */
f->msg->refcnt = 1;
}
else {
f->nread = 0;
f->off = 0;
f->into_body = body_available;
fq_debug(FQ_DEBUG_MSG, "incomplete message... (%d needed)\n",
(int)f->msg->payload_len - (int)f->into_body);
return 0;
}
}
return 0;
}
static void *
fq_conn_worker(void *u) {
int backoff = 0;
fq_conn_s *conn_s = (fq_conn_s *)u;
cmd_instr *last_entry = NULL;
uint16_t expect;
while(conn_s->stop == 0) {
restart:
expect = 0;
if(fq_client_connect_internal(conn_s) == 0) {
backoff = 0; /* we're good, restart our backoff */
}
while(conn_s->data_ready) {
hrtime_t t;
unsigned long long hb_us;
struct timeval tv;
int rv;
cmd_instr *entry;
ck_fifo_mpmc_entry_t *garbage;
while(ck_fifo_mpmc_dequeue(&conn_s->cmdq, &entry, &garbage) == true) {
free(garbage);
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "client acting on user req 0x%04x\n", entry->cmd);
#endif
switch(entry->cmd) {
case FQ_PROTO_STATUSREQ:
if(expect != 0) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
fq_debug(FQ_DEBUG_CONN, "sending status request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd)) {
free(entry);
goto restart;
}
expect = FQ_PROTO_STATUS;
last_entry = entry;
break;
case FQ_PROTO_HBREQ:
fq_debug(FQ_DEBUG_CONN, "sending heartbeat request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, entry->data.heartbeat.ms)) {
free(entry);
goto restart;
}
conn_s->cmd_hb_ms = entry->data.heartbeat.ms;
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_RCVTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_SNDTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
conn_s->cmd_hb_last = fq_gethrtime();
free(entry);
break;
case FQ_PROTO_BINDREQ:
{
unsigned short peermode = entry->data.bind->peermode ? 1 : 0;
if(expect != 0) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, peermode) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.bind->exchange.len,
entry->data.bind->exchange.name) < 0 ||
fq_write_short_cmd(conn_s->cmd_fd,
strlen(entry->data.bind->program),
entry->data.bind->program) < 0) {
goto restart;
}
expect = FQ_PROTO_BIND;
last_entry = entry;
}
break;
case FQ_PROTO_UNBINDREQ:
{
if(expect != 0) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint32(conn_s->cmd_fd, entry->data.unbind->route_id) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.unbind->exchange.len,
entry->data.unbind->exchange.name) < 0) {
goto restart;
}
expect = FQ_PROTO_UNBIND;
last_entry = entry;
}
break;
default:
if(conn_s->errorlog) conn_s->errorlog("unknown user-side cmd");
free(entry);
}
}
if(conn_s->cmd_hb_needed) {
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "-> heartbeat\n");
#endif
if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_HB)) break;
conn_s->cmd_hb_needed = 0;
}
rv = fq_client_wfrw_internal(conn_s->cmd_fd, 1, 0, 50, NULL);
t = fq_gethrtime();
hb_us = (unsigned long long)conn_s->cmd_hb_ms * 3 * 1000000ULL;
if(conn_s->cmd_hb_last && hb_us &&
conn_s->cmd_hb_last < (unsigned int) (t - hb_us)) {
char errbuf[256];
snprintf(errbuf, sizeof(errbuf), "heartbeat failed [%llu - %llu = %llu]",
(unsigned long long)t, (unsigned long long)conn_s->cmd_hb_last,
(unsigned long long)(t - conn_s->cmd_hb_last));
CONNERR(conn_s, errbuf);
break;
}
if(rv < 0) {
CONNERR(conn_s, strerror(errno));
break;
}
if(rv > 0) {
uint16_t hb;
if(fq_read_uint16(conn_s->cmd_fd, &hb)) break;
switch(hb) {
case FQ_PROTO_HB:
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "<- heartbeat\n");
#endif
conn_s->cmd_hb_last = fq_gethrtime();
conn_s->cmd_hb_needed = 1;
break;
case FQ_PROTO_STATUS:
if(expect != FQ_PROTO_STATUS) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_read_status(conn_s->cmd_fd,
last_entry->data.status.callback,
last_entry->data.status.closure))
goto restart;
expect = 0;
break;
case FQ_PROTO_BIND:
if(expect != FQ_PROTO_BIND) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.bind->out__route_id))
goto restart;
expect = 0;
break;
case FQ_PROTO_UNBIND:
if(expect != FQ_PROTO_UNBIND) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.unbind->out__success))
goto restart;
expect = 0;
break;
default:
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
break;
}
}
}
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "[cmd] connection failed: %s\n", conn_s->error);
#endif
usleep(backoff);
backoff += 10000;
}
fq_client_disconnect_internal(conn_s);
return (void *)NULL;
}
