int main (int argc, char *argv [])
{
int verbose = (argc > 1 && streq (argv [1], "-v"));
s_version_assert (2, 1);
s_catch_signals ();
broker_t *self = s_broker_new (verbose);
s_broker_bind (self, "tcp://*:5555");
// Get and process messages forever or until interrupted
while (!s_interrupted) {
zmq_pollitem_t items [] = {
{ self->socket, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, HEARTBEAT_INTERVAL * 1000);
// Process next input message, if any
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->socket);
if (self->verbose) {
s_console ("I: received message:");
zmsg_dump (msg);
}
char *sender = zmsg_pop (msg);
char *empty = zmsg_pop (msg);
char *header = zmsg_pop (msg);
if (streq (header, MDPC_CLIENT))
s_client_process (self, sender, msg);
else
if (streq (header, MDPW_WORKER))
s_worker_process (self, sender, msg);
else {
s_console ("E: invalid message:");
zmsg_dump (msg);
zmsg_destroy (&msg);
}
free (sender);
free (empty);
free (header);
}
// Disconnect and delete any expired workers
// Send heartbeats to idle workers if needed
if (s_clock () > self->heartbeat_at) {
s_broker_purge_workers (self);
worker_t *worker = zlist_first (self->waiting);
while (worker) {
s_worker_send (self, worker, MDPW_HEARTBEAT, NULL, NULL);
worker = zlist_next (self->waiting);
}
self->heartbeat_at = s_clock () + HEARTBEAT_INTERVAL;
}
}
if (s_interrupted)
printf ("W: interrupt received, shutting down...\n");
s_broker_destroy (&self);
return 0;
}
void
bind (std::string endpoint)
{
m_endpoint = endpoint;
m_socket->bind(m_endpoint.c_str());
s_console ("I: MDP broker/0.1.1 is active at %s", endpoint.c_str());
}
// Get and process messages forever or until interrupted
void
start_brokering() {
while (!s_interrupted) {
zmq::pollitem_t items [] = {
{ *m_socket, 0, ZMQ_POLLIN, 0 } };
zmq::poll (items, 1, HEARTBEAT_INTERVAL);
// Process next input message, if any
if (items [0].revents & ZMQ_POLLIN) {
zmsg *msg = new zmsg(*m_socket);
if (m_verbose) {
s_console ("I: received message:");
msg->dump ();
}
std::string sender = std::string((char*)msg->pop_front ().c_str());
msg->pop_front (); //empty message
std::string header = std::string((char*)msg->pop_front ().c_str());
// std::cout << "sbrok, sender: "<< sender << std::endl;
// std::cout << "sbrok, header: "<< header << std::endl;
// std::cout << "msg size: " << msg->parts() << std::endl;
// msg->dump();
if (header.compare(MDPC_CLIENT) == 0) {
client_process (sender, msg);
}
else if (header.compare(MDPW_WORKER) == 0) {
worker_process (sender, msg);
}
else {
s_console ("E: invalid message:");
msg->dump ();
delete msg;
}
}
// Disconnect and delete any expired workers
// Send heartbeats to idle workers if needed
if (s_clock () > m_heartbeat_at) {
purge_workers ();
for (std::vector<worker*>::iterator it = m_waiting.begin();
it != m_waiting.end() && (*it)!=0; it++) {
worker_send (*it, (char*)MDPW_HEARTBEAT, "", NULL);
}
m_heartbeat_at = s_clock () + HEARTBEAT_INTERVAL;
}
}
}
void
worker_process (std::string sender, zmsg *msg)
{
assert (msg && msg->parts() >= 1); // At least, command
std::string command = (char *)msg->pop_front().c_str();
bool worker_ready = m_workers.count(sender)>0;
worker *wrk = worker_require (sender);
if (command.compare (MDPW_READY) == 0) {
if (worker_ready) { // Not first command in session
worker_delete (wrk, 1);
}
else {
if (sender.size() >= 4 // Reserved service name
&& sender.find_first_of("mmi.") == 0) {
worker_delete (wrk, 1);
} else {
// Attach worker to service and mark as idle
std::string service_name = (char*)msg->pop_front ().c_str();
wrk->m_service = service_require (service_name);
wrk->m_service->m_workers++;
worker_waiting (wrk);
}
}
} else {
if (command.compare (MDPW_REPLY) == 0) {
if (worker_ready) {
// Remove & save client return envelope and insert the
// protocol header and service name, then rewrap envelope.
std::string client = msg->unwrap ();
msg->wrap (MDPC_CLIENT, wrk->m_service->m_name.c_str());
msg->wrap (client.c_str(), "");
msg->send (*m_socket);
worker_waiting (wrk);
}
else {
worker_delete (wrk, 1);
}
} else {
if (command.compare (MDPW_HEARTBEAT) == 0) {
if (worker_ready) {
wrk->m_expiry = s_clock () + HEARTBEAT_EXPIRY;
} else {
worker_delete (wrk, 1);
}
} else {
if (command.compare (MDPW_DISCONNECT) == 0) {
worker_delete (wrk, 0);
} else {
s_console ("E: invalid input message (%d)", (int) *command.c_str());
msg->dump ();
}
}
}
}
delete msg;
}
static void
s_worker_process (broker_t *self, char *sender, zmsg_t *msg)
{
assert (zmsg_parts (msg) >= 1); // At least, command
char *command = zmsg_pop (msg);
int worker_ready = (zhash_lookup (self->workers, sender) != NULL);
worker_t *worker = s_worker_require (self, sender);
if (streq (command, MDPW_READY)) {
if (worker_ready) // Not first command in session
s_worker_delete (self, worker, 1);
else
if (strlen (sender) >= 4 // Reserved service name
&& memcmp (sender, "mmi.", 4) == 0)
s_worker_delete (self, worker, 1);
else {
// Attach worker to service and mark as idle
char *service_name = zmsg_pop (msg);
worker->service = s_service_require (self, service_name);
worker->service->workers++;
s_worker_waiting (self, worker);
free (service_name);
}
}
else
if (streq (command, MDPW_REPLY)) {
if (worker_ready) {
// Remove & save client return envelope and insert the
// protocol header and service name, then rewrap envelope.
char *client = zmsg_unwrap (msg);
zmsg_wrap (msg, MDPC_CLIENT, worker->service->name);
zmsg_wrap (msg, client, "");
free (client);
zmsg_send (&msg, self->socket);
s_worker_waiting (self, worker);
}
else
s_worker_delete (self, worker, 1);
}
else
if (streq (command, MDPW_HEARTBEAT)) {
if (worker_ready)
worker->expiry = s_clock () + HEARTBEAT_EXPIRY;
else
s_worker_delete (self, worker, 1);
}
else
if (streq (command, MDPW_DISCONNECT))
s_worker_delete (self, worker, 0);
else {
s_console ("E: invalid input message (%d)", (int) *command);
zmsg_dump (msg);
}
free (command);
zmsg_destroy (&msg);
}
void s_mdcli_connect_to_broker (mdcli_t *self)
{
if (self->client)
zmq_close (self->client);
self->client = zmq_socket (self->context, ZMQ_REQ);
int linger = 0;
zmq_setsockopt (self->client, ZMQ_LINGER, &linger, sizeof (linger));
zmq_connect (self->client, self->broker);
if (self->verbose)
s_console ("I: connecting to broker at %s...", self->broker);
}
zmsg_t *
mdcli_recv (mdcli_t *self)
{
assert (self);
// Poll socket for a reply, with timeout
zmq_pollitem_t items [] = { { self->client, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, self->timeout * 1000);
// If we got a reply, process it
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->client);
if (self->verbose) {
s_console ("I: received reply:");
zmsg_dump (msg);
}
// Don't try to handle errors, just assert noisily
assert (zmsg_parts (msg) >= 4);
char *empty = zmsg_pop (msg);
assert (streq (empty, ""));
free (empty);
char *header = zmsg_pop (msg);
assert (streq (header, MDPC_CLIENT));
free (header);
char *service = zmsg_pop (msg);
assert (streq (service, service));
free (service);
return msg; // Success
}
if (s_interrupted)
printf ("W: interrupt received, killing client...\n");
else
if (self->verbose)
s_console ("W: permanent error, abandoning request");
return NULL;
}
// ---------------------------------------------------------------------
// Connect or reconnect to broker
void connect_to_broker ()
{
if (m_client) {
delete m_client;
}
m_client = new zmq::socket_t (*m_context, ZMQ_REQ);
int linger = 0;
m_client->setsockopt(ZMQ_LINGER, &linger, sizeof (linger));
//zmq_setsockopt (client, ZMQ_LINGER, &linger, sizeof (linger));
m_client->connect (m_broker.c_str());
if (m_verbose) {
s_console ("I: connecting to broker at %s...", m_broker.c_str());
}
}
static void
s_broker_purge_workers (broker_t *self)
{
worker_t *worker = zlist_first (self->waiting);
while (worker) {
if (!s_worker_expired (worker))
break; // Worker is alive, we're done here
if (self->verbose)
s_console ("I: deleting expired worker: %s",
worker->identity);
s_worker_delete (self, worker, 0);
worker = zlist_first (self->waiting);
}
}
service *
service_require (std::string name)
{
assert (name.size()>0);
if (m_services.count(name)) {
return m_services.at(name);
} else {
service * srv = new service(name);
m_services.insert(std::make_pair(name, srv));
if (m_verbose) {
s_console ("I: received message:");
}
return srv;
}
}
worker *
worker_require (std::string identity)
{
assert (identity.length()!=0);
// self->workers is keyed off worker identity
if (m_workers.count(identity)) {
return m_workers.at(identity);
} else {
worker *wrk = new worker(identity);
m_workers.insert(std::make_pair(identity, wrk));
if (m_verbose) {
s_console ("I: registering new worker: %s", identity.c_str());
}
return wrk;
}
}
static worker_t *
s_worker_require (broker_t *self, char *identity)
{
assert (identity);
// self->workers is keyed off worker identity
worker_t *worker =
(worker_t *) zhash_lookup (self->workers, identity);
if (worker == NULL) {
worker = (worker_t *) calloc (1, sizeof (worker_t));
worker->identity = strdup (identity);
zhash_insert (self->workers, identity, worker);
zhash_freefn (self->workers, identity, s_worker_destroy);
if (self->verbose)
s_console ("I: registering new worker: %s", identity);
}
return worker;
}
static service_t *
s_service_require (broker_t *self, char *name)
{
assert (name);
service_t *service =
(service_t *) zhash_lookup (self->services, name);
if (service == NULL) {
service = (service_t *) calloc (1, sizeof (service_t));
service->name = strdup (name);
service->requests = zlist_new ();
service->waiting = zlist_new ();
zhash_insert (self->services, name, service);
zhash_freefn (self->services, name, s_service_destroy);
if (self->verbose)
s_console ("I: received message:");
}
return service;
}
void
purge_workers ()
{
int64_t now = s_clock();
for (size_t i = 0; i < m_waiting.size();)
{
worker* wrk = m_waiting[i];
if (wrk->m_expiry > now)
{
++i;
continue;
}
if (m_verbose) {
s_console ("I: deleting expired worker: %s",
wrk->m_identity.c_str());
}
worker_delete (wrk, 0);
}
}
void connect_to_broker ()
{
if (m_worker) {
delete m_worker;
}
m_worker = new zmq::socket_t (*m_context, ZMQ_DEALER);
int linger = 0;
m_worker->setsockopt (ZMQ_LINGER, &linger, sizeof (linger));
m_worker->connect (m_broker.c_str());
if (m_verbose)
s_console ("I: connecting to broker at %s...", m_broker.c_str());
// Register service with broker
send_to_broker ((char*)MDPW_READY, m_service, NULL);
// If liveness hits zero, queue is considered disconnected
m_liveness = HEARTBEAT_LIVENESS;
m_heartbeat_at = s_clock () + m_heartbeat;
}
// ---------------------------------------------------------------------
// Send message to broker
// If no _msg is provided, creates one internally
void send_to_broker(char *command, std::string option, zmsg *_msg)
{
zmsg *msg = _msg? new zmsg(*_msg): new zmsg ();
// Stack protocol envelope to start of message
if (option.length() != 0) {
msg->push_front ((char*)option.c_str());
}
msg->push_front (command);
msg->push_front ((char*)MDPW_WORKER);
msg->push_front ((char*)"");
if (m_verbose) {
s_console ("I: sending %s to broker",
mdps_commands [(int) *command]);
msg->dump ();
}
msg->send (*m_worker);
}
int
mdcli_send (mdcli_t *self, char *service, zmsg_t **request_p)
{
assert (self);
assert (request_p);
zmsg_t *request = *request_p;
// Prefix request with protocol frames
// Frame 0: empty (REQ emulation)
// Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
// Frame 2: Service name (printable string)
zmsg_push (request, service);
zmsg_push (request, MDPC_CLIENT);
zmsg_push (request, "");
if (self->verbose) {
s_console ("I: send request to '%s' service:", service);
zmsg_dump (request);
}
zmsg_send (&request, self->client);
return 0;
}
static void
s_worker_send (
broker_t *self, worker_t *worker,
char *command, char *option, zmsg_t *msg)
{
msg = msg? zmsg_dup (msg): zmsg_new (NULL);
// Stack protocol envelope to start of message
if (option) // Optional frame after command
zmsg_push (msg, option);
zmsg_push (msg, command);
zmsg_push (msg, MDPW_WORKER);
// Stack routing envelope to start of message
zmsg_wrap (msg, worker->identity, "");
if (self->verbose) {
s_console ("I: sending %s to worker",
mdps_commands [(int) *command]);
zmsg_dump (msg);
}
zmsg_send (&msg, self->socket);
}
void
worker_send (worker *worker,
char *command, std::string option, zmsg *msg)
{
msg = (msg ? new zmsg(*msg) : new zmsg ());
// Stack protocol envelope to start of message
if (option.size()>0) { // Optional frame after command
msg->push_front ((char*)option.c_str());
}
msg->push_front (command);
msg->push_front ((char*)MDPW_WORKER);
// Stack routing envelope to start of message
msg->wrap(worker->m_identity.c_str(), "");
if (m_verbose) {
s_console ("I: sending %s to worker",
mdps_commands [(int) *command]);
msg->dump ();
}
msg->send (*m_socket);
delete msg;
}
void
s_broker_bind (broker_t *self, char *endpoint)
{
zmq_bind (self->socket, endpoint);
s_console ("I: MDP broker/0.1.1 is active at %s", endpoint);
}
zmsg *
send (std::string service, zmsg *&request_p)
{
assert (request_p);
zmsg *request = request_p;
// Prefix request with protocol frames
// Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
// Frame 2: Service name (printable string)
request->push_front((char*)service.c_str());
request->push_front((char*)MDPC_CLIENT);
if (m_verbose) {
s_console ("I: send request to '%s' service:", service.c_str());
request->dump();
}
int retries_left = m_retries;
while (retries_left && !s_interrupted) {
zmsg * msg = new zmsg(*request);
msg->send(*m_client);
while (!s_interrupted) {
// Poll socket for a reply, with timeout
zmq::pollitem_t items [] = {
{ *m_client, 0, ZMQ_POLLIN, 0 } };
zmq::poll (items, 1, m_timeout);
// If we got a reply, process it
if (items [0].revents & ZMQ_POLLIN) {
zmsg * recv_msg = new zmsg(*m_client);
if (m_verbose) {
s_console ("I: received reply:");
recv_msg->dump ();
}
// Don't try to handle errors, just assert noisily
assert (recv_msg->parts () >= 3);
std::basic_string<unsigned char> header = recv_msg->pop_front();
assert (header.compare((unsigned char *)MDPC_CLIENT) == 0);
std::basic_string<unsigned char> reply_service = recv_msg->pop_front();
assert (reply_service.compare((unsigned char *)service.c_str()) == 0);
delete request;
return recv_msg; // Success
}
else {
if (--retries_left) {
if (m_verbose) {
s_console ("W: no reply, reconnecting...");
}
// Reconnect, and resend message
connect_to_broker ();
zmsg msg (*request);
msg.send (*m_client);
}
else {
if (m_verbose) {
s_console ("W: permanent error, abandoning request");
}
break; // Give up
}
}
}
}
if (s_interrupted) {
std::cout << "W: interrupt received, killing client..." << std::endl;
}
delete request;
return 0;
}
zmsg *
recv (zmsg *&reply_p)
{
// Format and send the reply if we were provided one
zmsg *reply = reply_p;
assert (reply || !m_expect_reply);
if (reply) {
assert (m_reply_to.size()!=0);
reply->wrap (m_reply_to.c_str(), "");
m_reply_to = "";
send_to_broker ((char*)MDPW_REPLY, "", reply);
delete reply_p;
reply_p = 0;
}
m_expect_reply = true;
while (!s_interrupted) {
zmq::pollitem_t items [] = {
{ *m_worker, 0, ZMQ_POLLIN, 0 } };
zmq::poll (items, 1, m_heartbeat * 1000);
if (items [0].revents & ZMQ_POLLIN) {
zmsg *msg = new zmsg(*m_worker);
if (m_verbose) {
s_console ("I: received message from broker:");
msg->dump ();
}
m_liveness = HEARTBEAT_LIVENESS;
// Don't try to handle errors, just assert noisily
assert (msg->parts () >= 3);
std::basic_string<unsigned char> empty = msg->pop_front ();
assert (empty.compare((unsigned char *)"") == 0);
//assert (strcmp (empty, "") == 0);
//free (empty);
std::basic_string<unsigned char> header = msg->pop_front ();
assert (header.compare((unsigned char *)MDPW_WORKER) == 0);
//free (header);
std::string command = (char*) msg->pop_front ().c_str();
if (command.compare (MDPW_REQUEST) == 0) {
// We should pop and save as many addresses as there are
// up to a null part, but for now, just save one...
m_reply_to = msg->unwrap ();
return msg; // We have a request to process
}
else if (command.compare (MDPW_HEARTBEAT) == 0) {
// Do nothing for heartbeats
}
else if (command.compare (MDPW_DISCONNECT) == 0) {
connect_to_broker ();
}
else {
s_console ("E: invalid input message (%d)",
(int) *(command.c_str()));
msg->dump ();
}
delete msg;
}
else
if (--m_liveness == 0) {
if (m_verbose) {
s_console ("W: disconnected from broker - retrying...");
}
s_sleep (m_reconnect);
connect_to_broker ();
}
// Send HEARTBEAT if it's time
if (s_clock () > m_heartbeat_at) {
send_to_broker ((char*)MDPW_HEARTBEAT, "", NULL);
m_heartbeat_at = s_clock () + m_heartbeat;
}
}
if (s_interrupted)
printf ("W: interrupt received, killing worker...\n");
return NULL;
}
zmsg_t *
mdcli_send (mdcli_t *self, char *service, zmsg_t **request_p)
{
assert (self);
assert (request_p);
zmsg_t *request = *request_p;
// Prefix request with protocol frames
// Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
// Frame 2: Service name (printable string)
zmsg_push (request, service);
zmsg_push (request, MDPC_CLIENT);
if (self->verbose) {
s_console ("I: send request to '%s' service:", service);
zmsg_dump (request);
}
int retries_left = self->retries;
while (retries_left && !s_interrupted) {
zmsg_t *msg = zmsg_dup (request);
zmsg_send (&msg, self->client);
while (!s_interrupted) {
// Poll socket for a reply, with timeout
zmq_pollitem_t items [] = {
{ self->client, 0, ZMQ_POLLIN, 0 } };
zmq_poll (items, 1, self->timeout * 1000);
// If we got a reply, process it
if (items [0].revents & ZMQ_POLLIN) {
zmsg_t *msg = zmsg_recv (self->client);
if (self->verbose) {
s_console ("I: received reply:");
zmsg_dump (msg);
}
// Don't try to handle errors, just assert noisily
assert (zmsg_parts (msg) >= 3);
char *header = zmsg_pop (msg);
assert (streq (header, MDPC_CLIENT));
free (header);
char *reply_service = zmsg_pop (msg);
assert (streq (reply_service, service));
free (reply_service);
zmsg_destroy (&request);
return msg; // Success
}
else
if (--retries_left) {
if (self->verbose)
s_console ("W: no reply, reconnecting...");
// Reconnect, and resend message
s_mdcli_connect_to_broker (self);
zmsg_t *msg = zmsg_dup (request);
zmsg_send (&msg, self->client);
}
else {
if (self->verbose)
s_console ("W: permanent error, abandoning request");
break; // Give up
}
}
}
if (s_interrupted)
printf ("W: interrupt received, killing client...\n");
zmsg_destroy (&request);
return NULL;
}
