static int
fq_client_do_auth(fq_conn_s *conn_s) {
int len, qlen, qtlen;
uint16_t cmd;
char error[1024];
char queue_composed[MAX_RK_LEN*2 + 1];
if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_AUTH_CMD)) return -1;
if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_AUTH_PLAIN)) return -2;
if(fq_write_short_cmd(conn_s->cmd_fd, strlen(conn_s->user), conn_s->user) < 0)
return -3;
qlen = strlen(conn_s->queue);
if(qlen > MAX_RK_LEN) qlen = MAX_RK_LEN;
qtlen = strlen(conn_s->queue_type);
if(qtlen > MAX_RK_LEN) qtlen = MAX_RK_LEN;
len = qlen + qtlen + 1;
memcpy(queue_composed, conn_s->queue, qlen);
queue_composed[qlen] = '\0';
memcpy(queue_composed + qlen + 1, conn_s->queue_type, qtlen);
if(fq_write_short_cmd(conn_s->cmd_fd, len, queue_composed) < 0)
return -4;
if(fq_write_short_cmd(conn_s->cmd_fd, strlen(conn_s->pass), conn_s->pass) < 0)
return -5;
if(fq_read_uint16(conn_s->cmd_fd, &cmd)) return -6;
switch(cmd) {
case FQ_PROTO_ERROR:
len = fq_read_short_cmd(conn_s->cmd_fd, sizeof(error)-1, error);
if(conn_s->errorlog) {
if(len > (int)sizeof(error)-1) len = sizeof(error)-1;
if(len < 0) conn_s->errorlog(conn_s, "error reading error");
else conn_s->errorlog(conn_s,error);
}
return -7;
case FQ_PROTO_AUTH_RESP:
len = fq_read_short_cmd(conn_s->cmd_fd,
sizeof(conn_s->key.name), conn_s->key.name);
if(len < 0 || len > (int)sizeof(conn_s->key.name)) return -8;
conn_s->key.len = len;
#ifdef DEBUG
{
char hex[260];
if(fq_rk_to_hex(hex, sizeof(hex), &conn_s->key) >= 0)
fq_debug(FQ_DEBUG_CONN, "client keyed:\n%s\n", hex);
}
#endif
conn_s->data_ready = 1;
break;
default:
if(conn_s->errorlog) {
snprintf(error, sizeof(error),
"server auth response 0x%04x unknown\n", cmd);
conn_s->errorlog(conn_s, error);
}
return -9;
}
return 0;
}
static int
fq_client_data_connect_internal(fq_conn_s *conn_s) {
int flags;
uint32_t cmd = htonl(conn_s->peermode ? FQ_PROTO_PEER_MODE
: FQ_PROTO_DATA_MODE);
/* We don't support data connections when the cmd connection is down */
if(conn_s->cmd_fd < 0) return -1;
if(conn_s->data_fd >= 0) {
int toclose = conn_s->data_fd;
conn_s->data_fd = -1;
fq_debug(FQ_DEBUG_CONN, "close(data_fd)\n");
close(toclose);
}
conn_s->data_fd = fq_socket_connect(conn_s);
if(conn_s->data_fd < 0) goto shutdown;
fq_debug(FQ_DEBUG_CONN, "connect(data_fd) -> %d\n", conn_s->data_fd);
if(write(conn_s->data_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
goto shutdown;
if(conn_s->peermode) {
if(write(conn_s->data_fd, &conn_s->peermode,
sizeof(conn_s->peermode)) != sizeof(conn_s->peermode))
goto shutdown;
}
#ifdef DEBUG
{
char hex[260];
if(fq_rk_to_hex(hex, sizeof(hex), &conn_s->key) >= 0)
fq_debug(FQ_DEBUG_CONN, "client keying:\n%s\n", hex);
}
#endif
if(fq_write_short_cmd(conn_s->data_fd,
conn_s->key.len, conn_s->key.name) < 0) {
goto shutdown;
}
conn_s->tosend_offset = 0;
if(((flags = fcntl(conn_s->data_fd, F_GETFL, 0)) == -1) ||
(fcntl(conn_s->data_fd, F_SETFL, flags | O_NONBLOCK) == -1))
goto shutdown;
return 0;
shutdown:
if(conn_s->data_fd >= 0) {
int toclose = conn_s->data_fd;
conn_s->data_fd = -1;
fq_debug(FQ_DEBUG_CONN, "close(data_fd)\n");
close(toclose);
}
return -1;
}
static void *
fq_conn_worker(void *u) {
int backoff = 0, i;
bool zero;
fq_conn_s *conn_s = (fq_conn_s *)u;
cmd_instr *last_entries[MAX_PENDING] = { NULL };
int last_entry_idx = 0;
int next_entry_idx = 0;
#define SAVE_ENTRY(e) do { \
if(last_entries[next_entry_idx] != NULL) { \
CONNERR(conn_s, "exceed max cmd pipeline"); \
goto restart; \
} \
last_entries[next_entry_idx++] = e; \
next_entry_idx = next_entry_idx % MAX_PENDING; \
} while(0)
#define last_entry last_entries[last_entry_idx]
#define PROCESS_ENTRY(e, should_free) do { \
fq_assert(last_entries[last_entry_idx] == e); \
if(should_free) free(last_entries[last_entry_idx]); \
last_entries[last_entry_idx++] = NULL; \
last_entry_idx = last_entry_idx % MAX_PENDING; \
} while(0)
cmd_instr *entry;
ck_pr_inc_uint(&conn_s->thrcnt);
while(conn_s->stop == 0) {
int wait_ms = 50;
if(fq_client_connect_internal(conn_s) == 0) {
backoff = 0; /* we're good, restart our backoff */
}
while(conn_s->data_ready && conn_s->stop == 0) {
hrtime_t t;
unsigned long long hb_us;
struct timeval tv;
int rv;
ck_fifo_spsc_dequeue_lock(&conn_s->cmdq);
while(ck_fifo_spsc_dequeue(&conn_s->cmdq, &entry) == true) {
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "client acting on user req 0x%04x\n", entry->cmd);
#endif
switch(entry->cmd) {
case FQ_PROTO_STATUSREQ:
fq_debug(FQ_DEBUG_CONN, "sending status request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd)) {
free(entry);
CONNERR(conn_s, "write failed (statusreq)");
goto restart;
}
SAVE_ENTRY(entry);
break;
case FQ_PROTO_HBREQ:
fq_debug(FQ_DEBUG_CONN, "sending heartbeat request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, entry->data.heartbeat.ms)) {
free(entry);
CONNERR(conn_s, "write failed (hbreq)");
goto restart;
}
conn_s->cmd_hb_ms = entry->data.heartbeat.ms;
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_RCVTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_SNDTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
conn_s->cmd_hb_last = fq_gethrtime();
free(entry);
break;
case FQ_PROTO_BINDREQ:
{
unsigned short flags = entry->data.bind->flags;
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, flags) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.bind->exchange.len,
entry->data.bind->exchange.name) < 0 ||
fq_write_short_cmd(conn_s->cmd_fd,
strlen(entry->data.bind->program),
entry->data.bind->program) < 0) {
CONNERR(conn_s, "write failed (bindreq)");
goto restart;
}
SAVE_ENTRY(entry);
}
break;
case FQ_PROTO_UNBINDREQ:
{
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint32(conn_s->cmd_fd, entry->data.unbind->route_id) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.unbind->exchange.len,
entry->data.unbind->exchange.name) < 0) {
CONNERR(conn_s, "write failed (unbindreq)");
goto restart;
}
SAVE_ENTRY(entry);
}
break;
default:
CONNERR(conn_s, "unknown user-side cmd");
free(entry);
}
}
ck_fifo_spsc_dequeue_unlock(&conn_s->cmdq);
if(conn_s->cmd_hb_needed) {
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "-> heartbeat\n");
#endif
if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_HB)) {
CONNERR(conn_s, "write failed (hb)");
break;
}
conn_s->cmd_hb_needed = 0;
}
rv = fq_client_wfrw_internal(conn_s->cmd_fd, 1, 0, wait_ms, NULL);
if(rv == 0) {
wait_ms = (wait_ms >> 2) + wait_ms;
if(conn_s->cmd_hb_ms && wait_ms > conn_s->cmd_hb_ms)
wait_ms = conn_s->cmd_hb_ms;
if(wait_ms > 1000) wait_ms = 1000;
}
else wait_ms = 50;
fq_debug(FQ_DEBUG_CONN, "fq_client_wfrw_internal(cmd:%d) -> %d\n", conn_s->cmd_fd, rv);
t = fq_gethrtime();
hb_us = (unsigned long long)conn_s->cmd_hb_ms * 3 * 1000000ULL;
if(conn_s->cmd_hb_last && hb_us &&
conn_s->cmd_hb_last < (unsigned int) (t - hb_us)) {
char errbuf[256];
snprintf(errbuf, sizeof(errbuf), "heartbeat failed [%llu - %llu = %llu]",
(unsigned long long)t, (unsigned long long)conn_s->cmd_hb_last,
(unsigned long long)(t - conn_s->cmd_hb_last));
CONNERR(conn_s, errbuf);
break;
}
if(rv < 0) {
CONNERR(conn_s, strerror(errno));
break;
}
if(rv > 0) {
bool handled = false;
uint16_t hb;
if(fq_read_uint16(conn_s->cmd_fd, &hb)) break;
switch(hb) {
case FQ_PROTO_HB:
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "<- heartbeat\n");
#endif
conn_s->cmd_hb_last = fq_gethrtime();
conn_s->cmd_hb_needed = 1;
break;
case FQ_PROTO_STATUS:
if(!last_entry || last_entry->cmd != FQ_PROTO_STATUSREQ) {
CONNERR(conn_s, "protocol violation (status unexpected)");
goto restart;
}
if(fq_read_status(conn_s->cmd_fd,
last_entry->data.status.callback,
last_entry->data.status.closure))
goto restart;
PROCESS_ENTRY(last_entry, 1);
break;
case FQ_PROTO_BIND:
if(!last_entry || last_entry->cmd != FQ_PROTO_BINDREQ) {
CONNERR(conn_s, "protocol violation (bind unexpected)");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.bind->out__route_id)) {
if(conn_s->bind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
}
else conn_s->bind_hook((fq_client)conn_s, last_entry->data.bind);
}
CONNERR(conn_s, "read failed (bind)");
goto restart;
}
if(conn_s->bind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
}
else conn_s->bind_hook((fq_client)conn_s, last_entry->data.bind);
}
if(!handled) PROCESS_ENTRY(last_entry, 1);
break;
case FQ_PROTO_UNBIND:
if(!last_entry || last_entry->cmd != FQ_PROTO_UNBINDREQ) {
CONNERR(conn_s, "protocol violation (unbind unexpected)");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.unbind->out__success)) {
if(conn_s->unbind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
}
conn_s->unbind_hook((fq_client)conn_s, last_entry->data.unbind);
}
CONNERR(conn_s, "read failed (unbind)");
goto restart;
}
if(conn_s->unbind_hook) {
if(conn_s->sync_hooks) {
enqueue_cmd_hook_req(conn_s, last_entry);
PROCESS_ENTRY(last_entry, 0);
handled = true;
last_entry = NULL;
}
conn_s->unbind_hook((fq_client)conn_s, last_entry->data.unbind);
}
if(!handled) PROCESS_ENTRY(last_entry,1);
break;
default:
CONNERR(conn_s, "protocol violation");
goto restart;
break;
}
}
}
static void *
fq_conn_worker(void *u) {
int backoff = 0;
fq_conn_s *conn_s = (fq_conn_s *)u;
cmd_instr *last_entry = NULL;
uint16_t expect;
while(conn_s->stop == 0) {
restart:
expect = 0;
if(fq_client_connect_internal(conn_s) == 0) {
backoff = 0; /* we're good, restart our backoff */
}
while(conn_s->data_ready) {
hrtime_t t;
unsigned long long hb_us;
struct timeval tv;
int rv;
cmd_instr *entry;
ck_fifo_mpmc_entry_t *garbage;
while(ck_fifo_mpmc_dequeue(&conn_s->cmdq, &entry, &garbage) == true) {
free(garbage);
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "client acting on user req 0x%04x\n", entry->cmd);
#endif
switch(entry->cmd) {
case FQ_PROTO_STATUSREQ:
if(expect != 0) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
fq_debug(FQ_DEBUG_CONN, "sending status request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd)) {
free(entry);
goto restart;
}
expect = FQ_PROTO_STATUS;
last_entry = entry;
break;
case FQ_PROTO_HBREQ:
fq_debug(FQ_DEBUG_CONN, "sending heartbeat request\n");
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, entry->data.heartbeat.ms)) {
free(entry);
goto restart;
}
conn_s->cmd_hb_ms = entry->data.heartbeat.ms;
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_RCVTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_SNDTIMEO,
&tv, sizeof(tv)))
CONNERR(conn_s, strerror(errno));
conn_s->cmd_hb_last = fq_gethrtime();
free(entry);
break;
case FQ_PROTO_BINDREQ:
{
unsigned short peermode = entry->data.bind->peermode ? 1 : 0;
if(expect != 0) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint16(conn_s->cmd_fd, peermode) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.bind->exchange.len,
entry->data.bind->exchange.name) < 0 ||
fq_write_short_cmd(conn_s->cmd_fd,
strlen(entry->data.bind->program),
entry->data.bind->program) < 0) {
goto restart;
}
expect = FQ_PROTO_BIND;
last_entry = entry;
}
break;
case FQ_PROTO_UNBINDREQ:
{
if(expect != 0) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
fq_write_uint32(conn_s->cmd_fd, entry->data.unbind->route_id) ||
fq_write_short_cmd(conn_s->cmd_fd,
entry->data.unbind->exchange.len,
entry->data.unbind->exchange.name) < 0) {
goto restart;
}
expect = FQ_PROTO_UNBIND;
last_entry = entry;
}
break;
default:
if(conn_s->errorlog) conn_s->errorlog("unknown user-side cmd");
free(entry);
}
}
if(conn_s->cmd_hb_needed) {
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "-> heartbeat\n");
#endif
if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_HB)) break;
conn_s->cmd_hb_needed = 0;
}
rv = fq_client_wfrw_internal(conn_s->cmd_fd, 1, 0, 50, NULL);
t = fq_gethrtime();
hb_us = (unsigned long long)conn_s->cmd_hb_ms * 3 * 1000000ULL;
if(conn_s->cmd_hb_last && hb_us &&
conn_s->cmd_hb_last < (unsigned int) (t - hb_us)) {
char errbuf[256];
snprintf(errbuf, sizeof(errbuf), "heartbeat failed [%llu - %llu = %llu]",
(unsigned long long)t, (unsigned long long)conn_s->cmd_hb_last,
(unsigned long long)(t - conn_s->cmd_hb_last));
CONNERR(conn_s, errbuf);
break;
}
if(rv < 0) {
CONNERR(conn_s, strerror(errno));
break;
}
if(rv > 0) {
uint16_t hb;
if(fq_read_uint16(conn_s->cmd_fd, &hb)) break;
switch(hb) {
case FQ_PROTO_HB:
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "<- heartbeat\n");
#endif
conn_s->cmd_hb_last = fq_gethrtime();
conn_s->cmd_hb_needed = 1;
break;
case FQ_PROTO_STATUS:
if(expect != FQ_PROTO_STATUS) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_read_status(conn_s->cmd_fd,
last_entry->data.status.callback,
last_entry->data.status.closure))
goto restart;
expect = 0;
break;
case FQ_PROTO_BIND:
if(expect != FQ_PROTO_BIND) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.bind->out__route_id))
goto restart;
expect = 0;
break;
case FQ_PROTO_UNBIND:
if(expect != FQ_PROTO_UNBIND) {
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
}
if(fq_read_uint32(conn_s->cmd_fd,
&last_entry->data.unbind->out__success))
goto restart;
expect = 0;
break;
default:
if(conn_s->errorlog) conn_s->errorlog("protocol violation");
goto restart;
break;
}
}
}
#ifdef DEBUG
fq_debug(FQ_DEBUG_CONN, "[cmd] connection failed: %s\n", conn_s->error);
#endif
usleep(backoff);
backoff += 10000;
}
fq_client_disconnect_internal(conn_s);
return (void *)NULL;
}