void
agent_router_message (agent_t *self)
{
zmsg_t *reply = zmsg_recv (self->router);
// Frame 0 is server that replied
char *endpoint = zmsg_pop (reply);
server_t *server = (server_t *) zhash_lookup (self->servers, endpoint);
assert (server);
free (endpoint);
if (!server->alive) {
zlist_append (self->actives, server);
server->alive = 1;
}
server->ping_at = s_clock () + PING_INTERVAL;
server->expires = s_clock () + SERVER_TTL;
// Frame 1 may be sequence number for reply
if (zmsg_parts (reply) > 1
&& atoi (zmsg_address (reply)) == self->sequence) {
free (zmsg_pop (reply));
zmsg_push (reply, "OK");
zmsg_send (&reply, self->control);
zmsg_destroy (&self->request);
}
zmsg_destroy (&reply);
}
zmsg_t *
mdwrk_recv (mdwrk_t *self, zmsg_t *reply)
{
// Format and send the reply if we were provided one
assert (reply || !self->expect_reply);
if (reply) {
zmsg_t *msg = zmsg_dup (reply);
zmsg_push (msg, MDPS_REPLY);
zmsg_push (msg, MDPS_HEADER);
zmsg_send (&msg, self->worker);
}
self->expect_reply = 1;
while (1) {
zmq_pollitem_t items [] = { { self->worker, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, HEARTBEAT_INTERVAL * 1000);
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->worker);
self->liveness = HEARTBEAT_LIVENESS;
// Don't try to handle errors, just assert noisily
assert (zmsg_parts (msg) >= 3);
char *header = zmsg_pop (msg);
assert (strcmp (header, MDPS_HEADER) == 0);
free (header);
char *command = zmsg_pop (msg);
if (strcmp (command, MDPS_REQUEST) == 0)
return msg; // We have a request to process
else
if (strcmp (command, MDPS_HEARTBEAT) == 0)
; // Do nothing for heartbeats
else
if (strcmp (command, MDPS_DISCONNECT) == 0)
break; // Return empty handed
else {
printf ("E: invalid input message (%d)\n", (int) command [1]);
zmsg_dump (msg);
}
free (command);
}
else
if (--self->liveness == 0) {
s_sleep (RECONNECT_INTERVAL);
s_connect_to_broker (self);
}
// Send HEARTBEAT if it's time
if (s_clock () > self->heartbeat_at) {
self->heartbeat_at = s_clock () + HEARTBEAT_INTERVAL;
s_send (self->worker, "HEARTBEAT");
}
}
// We exit if we've been disconnected
return NULL;
}
static void
s_worker_process (broker_t *self, char *sender, zmsg_t *msg)
{
assert (zmsg_parts (msg) >= 1); // At least, command
char *command = zmsg_pop (msg);
int worker_ready = (zhash_lookup (self->workers, sender) != NULL);
worker_t *worker = s_worker_require (self, sender);
if (streq (command, MDPW_READY)) {
if (worker_ready) // Not first command in session
s_worker_delete (self, worker, 1);
else
if (strlen (sender) >= 4 // Reserved service name
&& memcmp (sender, "mmi.", 4) == 0)
s_worker_delete (self, worker, 1);
else {
// Attach worker to service and mark as idle
char *service_name = zmsg_pop (msg);
worker->service = s_service_require (self, service_name);
worker->service->workers++;
s_worker_waiting (self, worker);
free (service_name);
}
}
else
if (streq (command, MDPW_REPLY)) {
if (worker_ready) {
// Remove & save client return envelope and insert the
// protocol header and service name, then rewrap envelope.
char *client = zmsg_unwrap (msg);
zmsg_wrap (msg, MDPC_CLIENT, worker->service->name);
zmsg_wrap (msg, client, "");
free (client);
zmsg_send (&msg, self->socket);
s_worker_waiting (self, worker);
}
else
s_worker_delete (self, worker, 1);
}
else
if (streq (command, MDPW_HEARTBEAT)) {
if (worker_ready)
worker->expiry = s_clock () + HEARTBEAT_EXPIRY;
else
s_worker_delete (self, worker, 1);
}
else
if (streq (command, MDPW_DISCONNECT))
s_worker_delete (self, worker, 0);
else {
s_console ("E: invalid input message (%d)", (int) *command);
zmsg_dump (msg);
}
free (command);
zmsg_destroy (&msg);
}
zmsg_t *
mdcli_send (mdcli_t *self, char *service, zmsg_t *request)
{
int retries_left = REQUEST_RETRIES;
while (retries_left) {
// Prefix request with protocol frames
// Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
// Frame 2: Service name (printable string)
zmsg_t *msg = zmsg_dup (request);
zmsg_push (msg, service);
zmsg_push (msg, MDPC_HEADER);
zmsg_send (&msg, self->client);
while (1) {
// Poll socket for a reply, with timeout
zmq_pollitem_t items [] = { { self->client, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, REQUEST_TIMEOUT * 1000);
// If we got a reply, process it
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->client);
// Don't try to handle errors, just assert noisily
assert (zmsg_parts (msg) >= 3);
char *header = zmsg_pop (msg);
assert (strcmp (header, MDPC_HEADER) == 0);
free (header);
char *service = zmsg_pop (msg);
assert (strcmp (service, service) == 0);
free (service);
return msg; // Success
}
else
if (--retries_left) {
// Reconnect, and resend message
s_connect_to_broker (self);
zmsg_t *msg = zmsg_dup (request);
zmsg_push (msg, service);
zmsg_push (msg, MDPC_HEADER);
zmsg_send (&msg, self->client);
}
else
break; // Give up
}
}
return NULL;
}
static void
s_client_process (broker_t *self, char *sender, zmsg_t *msg)
{
assert (zmsg_parts (msg) >= 2); // Service name + body
char *service_name = zmsg_pop (msg);
service_t *service = s_service_require (self, service_name);
// Set reply return address to client sender
zmsg_wrap (msg, sender, "");
if (strlen (service_name) >= 4
&& memcmp (service_name, "mmi.", 4) == 0)
s_service_internal (self, service_name, msg);
else
s_service_dispatch (self, service, msg);
free (service_name);
}
zmsg_t *
mdcli_recv (mdcli_t *self)
{
assert (self);
// Poll socket for a reply, with timeout
zmq_pollitem_t items [] = { { self->client, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, self->timeout * 1000);
// If we got a reply, process it
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->client);
if (self->verbose) {
s_console ("I: received reply:");
zmsg_dump (msg);
}
// Don't try to handle errors, just assert noisily
assert (zmsg_parts (msg) >= 4);
char *empty = zmsg_pop (msg);
assert (streq (empty, ""));
free (empty);
char *header = zmsg_pop (msg);
assert (streq (header, MDPC_CLIENT));
free (header);
char *service = zmsg_pop (msg);
assert (streq (service, service));
free (service);
return msg; // Success
}
if (s_interrupted)
printf ("W: interrupt received, killing client...\n");
else
if (self->verbose)
s_console ("W: permanent error, abandoning request");
return NULL;
}
zmsg_t *
flclient_request (flclient_t *self, zmsg_t **request_p)
{
assert (self);
assert (*request_p);
zmsg_t *request = *request_p;
// Prefix request with sequence number and empty envelope
char sequence_text [10];
sprintf (sequence_text, "%u", ++self->sequence);
zmsg_push (request, sequence_text);
zmsg_push (request, "");
// Blast the request to all connected servers
int server;
for (server = 0; server < self->servers; server++) {
zmsg_t *msg = zmsg_dup (request);
zmsg_send (&msg, self->socket);
}
// Wait for a matching reply to arrive from anywhere
// Since we can poll several times, calculate each one
zmsg_t *reply = NULL;
uint64_t endtime = s_clock () + GLOBAL_TIMEOUT;
while (s_clock () < endtime) {
zmq_pollitem_t items [] = { { self->socket, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, (endtime - s_clock ()) * 1000);
if (items [0].revents & ZMQ_POLLIN) {
reply = zmsg_recv (self->socket);
assert (zmsg_parts (reply) == 3);
free (zmsg_pop (reply));
if (atoi (zmsg_address (reply)) == self->sequence)
break;
zmsg_destroy (&reply);
}
}
zmsg_destroy (request_p);
return reply;
}
// Accept a request and reply with the same text a random number of
// times, with random delays between replies.
//
static void *
server_worker (void *context) {
void *worker = zmq_socket (context, ZMQ_XREQ);
zmq_connect (worker, "inproc://backend");
while (1) {
// The XREQ socket gives us the address envelope and message
zmsg_t *msg = zmsg_recv (worker);
assert (zmsg_parts (msg) == 2);
// Send 0..4 replies back
int reply, replies = randof (5);
for (reply = 0; reply < replies; reply++) {
// Sleep for some fraction of a second
struct timespec t = { 0, randof (100000000) + 1 };
nanosleep (&t, NULL);
zmsg_t *dup = zmsg_dup (msg);
zmsg_send (&dup, worker);
}
zmsg_destroy (&msg);
}
zmq_close (worker);
return (NULL);
}
int main (void)
{
s_version_assert (2, 1);
srandom ((unsigned) time (NULL));
void *context = zmq_init (1);
void *worker = s_worker_socket (context);
// If liveness hits zero, queue is considered disconnected
size_t liveness = HEARTBEAT_LIVENESS;
size_t interval = INTERVAL_INIT;
// Send out heartbeats at regular intervals
uint64_t heartbeat_at = s_clock () + HEARTBEAT_INTERVAL;
int cycles = 0;
while (1) {
zmq_pollitem_t items [] = { { worker, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, HEARTBEAT_INTERVAL * 1000);
if (items [0].revents & ZMQ_POLLIN) {
// Get message
// - 3-part envelope + content -> request
// - 1-part "HEARTBEAT" -> heartbeat
zmsg_t *msg = zmsg_recv (worker);
if (zmsg_parts (msg) == 3) {
// Simulate various problems, after a few cycles
cycles++;
if (cycles > 3 && randof (5) == 0) {
printf ("I: (%s) simulating a crash\n", identity);
zmsg_destroy (&msg);
break;
}
else
if (cycles > 3 && randof (5) == 0) {
printf ("I: (%s) simulating CPU overload\n", identity);
sleep (5);
}
printf ("I: (%s) normal reply - %s\n",
identity, zmsg_body (msg));
zmsg_send (&msg, worker);
liveness = HEARTBEAT_LIVENESS;
sleep (1); // Do some heavy work
}
else
if (zmsg_parts (msg) == 1
&& strcmp (zmsg_body (msg), "HEARTBEAT") == 0)
liveness = HEARTBEAT_LIVENESS;
else {
printf ("E: (%s) invalid message\n", identity);
zmsg_dump (msg);
}
interval = INTERVAL_INIT;
}
else
if (--liveness == 0) {
printf ("W: (%s) heartbeat failure, can't reach queue\n",
identity);
printf ("W: (%s) reconnecting in %zd msec...\n",
identity, interval);
s_sleep (interval);
if (interval < INTERVAL_MAX)
interval *= 2;
zmq_close (worker);
worker = s_worker_socket (context);
liveness = HEARTBEAT_LIVENESS;
}
// Send heartbeat to queue if it's time
if (s_clock () > heartbeat_at) {
heartbeat_at = s_clock () + HEARTBEAT_INTERVAL;
printf ("I: (%s) worker heartbeat\n", identity);
s_send (worker, "HEARTBEAT");
}
}
zmq_close (worker);
zmq_term (context);
return 0;
}
zmsg_t *
mdcli_send (mdcli_t *self, char *service, zmsg_t **request_p)
{
assert (self);
assert (request_p);
zmsg_t *request = *request_p;
// Prefix request with protocol frames
// Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
// Frame 2: Service name (printable string)
zmsg_push (request, service);
zmsg_push (request, MDPC_CLIENT);
if (self->verbose) {
s_console ("I: send request to '%s' service:", service);
zmsg_dump (request);
}
int retries_left = self->retries;
while (retries_left && !s_interrupted) {
zmsg_t *msg = zmsg_dup (request);
zmsg_send (&msg, self->client);
while (!s_interrupted) {
// Poll socket for a reply, with timeout
zmq_pollitem_t items [] = {
{ self->client, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, self->timeout * 1000);
// If we got a reply, process it
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->client);
if (self->verbose) {
s_console ("I: received reply:");
zmsg_dump (msg);
}
// Don't try to handle errors, just assert noisily
assert (zmsg_parts (msg) >= 3);
char *header = zmsg_pop (msg);
assert (streq (header, MDPC_CLIENT));
free (header);
char *reply_service = zmsg_pop (msg);
assert (streq (reply_service, service));
free (reply_service);
zmsg_destroy (&request);
return msg; // Success
}
else
if (--retries_left) {
if (self->verbose)
s_console ("W: no reply, reconnecting...");
// Reconnect, and resend message
s_mdcli_connect_to_broker (self);
zmsg_t *msg = zmsg_dup (request);
zmsg_send (&msg, self->client);
}
else {
if (self->verbose)
s_console ("W: permanent error, abandoning request");
break; // Give up
}
}
}
if (s_interrupted)
printf ("W: interrupt received, killing client...\n");
zmsg_destroy (&request);
return NULL;
}
