int main (int argc, char *argv []) { int verbose = (argc > 1 && streq (argv [1], "-v")); s_version_assert (2, 1); s_catch_signals (); broker_t *self = s_broker_new (verbose); s_broker_bind (self, "tcp://*:5555"); // Get and process messages forever or until interrupted while (!s_interrupted) { zmq_pollitem_t items [] = { { self->socket, 0, ZMQ_POLLIN, 0 } }; zmq_poll (items, 1, HEARTBEAT_INTERVAL * 1000); // Process next input message, if any if (items [0].revents & ZMQ_POLLIN) { zmsg_t *msg = zmsg_recv (self->socket); if (self->verbose) { s_console ("I: received message:"); zmsg_dump (msg); } char *sender = zmsg_pop (msg); char *empty = zmsg_pop (msg); char *header = zmsg_pop (msg); if (streq (header, MDPC_CLIENT)) s_client_process (self, sender, msg); else if (streq (header, MDPW_WORKER)) s_worker_process (self, sender, msg); else { s_console ("E: invalid message:"); zmsg_dump (msg); zmsg_destroy (&msg); } free (sender); free (empty); free (header); } // Disconnect and delete any expired workers // Send heartbeats to idle workers if needed if (s_clock () > self->heartbeat_at) { s_broker_purge_workers (self); worker_t *worker = zlist_first (self->waiting); while (worker) { s_worker_send (self, worker, MDPW_HEARTBEAT, NULL, NULL); worker = zlist_next (self->waiting); } self->heartbeat_at = s_clock () + HEARTBEAT_INTERVAL; } } if (s_interrupted) printf ("W: interrupt received, shutting down...\n"); s_broker_destroy (&self); return 0; }
void bind (std::string endpoint) { m_endpoint = endpoint; m_socket->bind(m_endpoint.c_str()); s_console ("I: MDP broker/0.1.1 is active at %s", endpoint.c_str()); }
// Get and process messages forever or until interrupted void start_brokering() { while (!s_interrupted) { zmq::pollitem_t items [] = { { *m_socket, 0, ZMQ_POLLIN, 0 } }; zmq::poll (items, 1, HEARTBEAT_INTERVAL); // Process next input message, if any if (items [0].revents & ZMQ_POLLIN) { zmsg *msg = new zmsg(*m_socket); if (m_verbose) { s_console ("I: received message:"); msg->dump (); } std::string sender = std::string((char*)msg->pop_front ().c_str()); msg->pop_front (); //empty message std::string header = std::string((char*)msg->pop_front ().c_str()); // std::cout << "sbrok, sender: "<< sender << std::endl; // std::cout << "sbrok, header: "<< header << std::endl; // std::cout << "msg size: " << msg->parts() << std::endl; // msg->dump(); if (header.compare(MDPC_CLIENT) == 0) { client_process (sender, msg); } else if (header.compare(MDPW_WORKER) == 0) { worker_process (sender, msg); } else { s_console ("E: invalid message:"); msg->dump (); delete msg; } } // Disconnect and delete any expired workers // Send heartbeats to idle workers if needed if (s_clock () > m_heartbeat_at) { purge_workers (); for (std::vector<worker*>::iterator it = m_waiting.begin(); it != m_waiting.end() && (*it)!=0; it++) { worker_send (*it, (char*)MDPW_HEARTBEAT, "", NULL); } m_heartbeat_at = s_clock () + HEARTBEAT_INTERVAL; } } }
void worker_process (std::string sender, zmsg *msg) { assert (msg && msg->parts() >= 1); // At least, command std::string command = (char *)msg->pop_front().c_str(); bool worker_ready = m_workers.count(sender)>0; worker *wrk = worker_require (sender); if (command.compare (MDPW_READY) == 0) { if (worker_ready) { // Not first command in session worker_delete (wrk, 1); } else { if (sender.size() >= 4 // Reserved service name && sender.find_first_of("mmi.") == 0) { worker_delete (wrk, 1); } else { // Attach worker to service and mark as idle std::string service_name = (char*)msg->pop_front ().c_str(); wrk->m_service = service_require (service_name); wrk->m_service->m_workers++; worker_waiting (wrk); } } } else { if (command.compare (MDPW_REPLY) == 0) { if (worker_ready) { // Remove & save client return envelope and insert the // protocol header and service name, then rewrap envelope. std::string client = msg->unwrap (); msg->wrap (MDPC_CLIENT, wrk->m_service->m_name.c_str()); msg->wrap (client.c_str(), ""); msg->send (*m_socket); worker_waiting (wrk); } else { worker_delete (wrk, 1); } } else { if (command.compare (MDPW_HEARTBEAT) == 0) { if (worker_ready) { wrk->m_expiry = s_clock () + HEARTBEAT_EXPIRY; } else { worker_delete (wrk, 1); } } else { if (command.compare (MDPW_DISCONNECT) == 0) { worker_delete (wrk, 0); } else { s_console ("E: invalid input message (%d)", (int) *command.c_str()); msg->dump (); } } } } delete msg; }
static void s_worker_process (broker_t *self, char *sender, zmsg_t *msg) { assert (zmsg_parts (msg) >= 1); // At least, command char *command = zmsg_pop (msg); int worker_ready = (zhash_lookup (self->workers, sender) != NULL); worker_t *worker = s_worker_require (self, sender); if (streq (command, MDPW_READY)) { if (worker_ready) // Not first command in session s_worker_delete (self, worker, 1); else if (strlen (sender) >= 4 // Reserved service name && memcmp (sender, "mmi.", 4) == 0) s_worker_delete (self, worker, 1); else { // Attach worker to service and mark as idle char *service_name = zmsg_pop (msg); worker->service = s_service_require (self, service_name); worker->service->workers++; s_worker_waiting (self, worker); free (service_name); } } else if (streq (command, MDPW_REPLY)) { if (worker_ready) { // Remove & save client return envelope and insert the // protocol header and service name, then rewrap envelope. char *client = zmsg_unwrap (msg); zmsg_wrap (msg, MDPC_CLIENT, worker->service->name); zmsg_wrap (msg, client, ""); free (client); zmsg_send (&msg, self->socket); s_worker_waiting (self, worker); } else s_worker_delete (self, worker, 1); } else if (streq (command, MDPW_HEARTBEAT)) { if (worker_ready) worker->expiry = s_clock () + HEARTBEAT_EXPIRY; else s_worker_delete (self, worker, 1); } else if (streq (command, MDPW_DISCONNECT)) s_worker_delete (self, worker, 0); else { s_console ("E: invalid input message (%d)", (int) *command); zmsg_dump (msg); } free (command); zmsg_destroy (&msg); }
void s_mdcli_connect_to_broker (mdcli_t *self) { if (self->client) zmq_close (self->client); self->client = zmq_socket (self->context, ZMQ_REQ); int linger = 0; zmq_setsockopt (self->client, ZMQ_LINGER, &linger, sizeof (linger)); zmq_connect (self->client, self->broker); if (self->verbose) s_console ("I: connecting to broker at %s...", self->broker); }
zmsg_t * mdcli_recv (mdcli_t *self) { assert (self); // Poll socket for a reply, with timeout zmq_pollitem_t items [] = { { self->client, 0, ZMQ_POLLIN, 0 } }; zmq_poll (items, 1, self->timeout * 1000); // If we got a reply, process it if (items [0].revents & ZMQ_POLLIN) { zmsg_t *msg = zmsg_recv (self->client); if (self->verbose) { s_console ("I: received reply:"); zmsg_dump (msg); } // Don't try to handle errors, just assert noisily assert (zmsg_parts (msg) >= 4); char *empty = zmsg_pop (msg); assert (streq (empty, "")); free (empty); char *header = zmsg_pop (msg); assert (streq (header, MDPC_CLIENT)); free (header); char *service = zmsg_pop (msg); assert (streq (service, service)); free (service); return msg; // Success } if (s_interrupted) printf ("W: interrupt received, killing client...\n"); else if (self->verbose) s_console ("W: permanent error, abandoning request"); return NULL; }
// --------------------------------------------------------------------- // Connect or reconnect to broker void connect_to_broker () { if (m_client) { delete m_client; } m_client = new zmq::socket_t (*m_context, ZMQ_REQ); int linger = 0; m_client->setsockopt(ZMQ_LINGER, &linger, sizeof (linger)); //zmq_setsockopt (client, ZMQ_LINGER, &linger, sizeof (linger)); m_client->connect (m_broker.c_str()); if (m_verbose) { s_console ("I: connecting to broker at %s...", m_broker.c_str()); } }
static void s_broker_purge_workers (broker_t *self) { worker_t *worker = zlist_first (self->waiting); while (worker) { if (!s_worker_expired (worker)) break; // Worker is alive, we're done here if (self->verbose) s_console ("I: deleting expired worker: %s", worker->identity); s_worker_delete (self, worker, 0); worker = zlist_first (self->waiting); } }
service * service_require (std::string name) { assert (name.size()>0); if (m_services.count(name)) { return m_services.at(name); } else { service * srv = new service(name); m_services.insert(std::make_pair(name, srv)); if (m_verbose) { s_console ("I: received message:"); } return srv; } }
worker * worker_require (std::string identity) { assert (identity.length()!=0); // self->workers is keyed off worker identity if (m_workers.count(identity)) { return m_workers.at(identity); } else { worker *wrk = new worker(identity); m_workers.insert(std::make_pair(identity, wrk)); if (m_verbose) { s_console ("I: registering new worker: %s", identity.c_str()); } return wrk; } }
static worker_t * s_worker_require (broker_t *self, char *identity) { assert (identity); // self->workers is keyed off worker identity worker_t *worker = (worker_t *) zhash_lookup (self->workers, identity); if (worker == NULL) { worker = (worker_t *) calloc (1, sizeof (worker_t)); worker->identity = strdup (identity); zhash_insert (self->workers, identity, worker); zhash_freefn (self->workers, identity, s_worker_destroy); if (self->verbose) s_console ("I: registering new worker: %s", identity); } return worker; }
static service_t * s_service_require (broker_t *self, char *name) { assert (name); service_t *service = (service_t *) zhash_lookup (self->services, name); if (service == NULL) { service = (service_t *) calloc (1, sizeof (service_t)); service->name = strdup (name); service->requests = zlist_new (); service->waiting = zlist_new (); zhash_insert (self->services, name, service); zhash_freefn (self->services, name, s_service_destroy); if (self->verbose) s_console ("I: received message:"); } return service; }
void purge_workers () { int64_t now = s_clock(); for (size_t i = 0; i < m_waiting.size();) { worker* wrk = m_waiting[i]; if (wrk->m_expiry > now) { ++i; continue; } if (m_verbose) { s_console ("I: deleting expired worker: %s", wrk->m_identity.c_str()); } worker_delete (wrk, 0); } }
void connect_to_broker () { if (m_worker) { delete m_worker; } m_worker = new zmq::socket_t (*m_context, ZMQ_DEALER); int linger = 0; m_worker->setsockopt (ZMQ_LINGER, &linger, sizeof (linger)); m_worker->connect (m_broker.c_str()); if (m_verbose) s_console ("I: connecting to broker at %s...", m_broker.c_str()); // Register service with broker send_to_broker ((char*)MDPW_READY, m_service, NULL); // If liveness hits zero, queue is considered disconnected m_liveness = HEARTBEAT_LIVENESS; m_heartbeat_at = s_clock () + m_heartbeat; }
// --------------------------------------------------------------------- // Send message to broker // If no _msg is provided, creates one internally void send_to_broker(char *command, std::string option, zmsg *_msg) { zmsg *msg = _msg? new zmsg(*_msg): new zmsg (); // Stack protocol envelope to start of message if (option.length() != 0) { msg->push_front ((char*)option.c_str()); } msg->push_front (command); msg->push_front ((char*)MDPW_WORKER); msg->push_front ((char*)""); if (m_verbose) { s_console ("I: sending %s to broker", mdps_commands [(int) *command]); msg->dump (); } msg->send (*m_worker); }
int mdcli_send (mdcli_t *self, char *service, zmsg_t **request_p) { assert (self); assert (request_p); zmsg_t *request = *request_p; // Prefix request with protocol frames // Frame 0: empty (REQ emulation) // Frame 1: "MDPCxy" (six bytes, MDP/Client x.y) // Frame 2: Service name (printable string) zmsg_push (request, service); zmsg_push (request, MDPC_CLIENT); zmsg_push (request, ""); if (self->verbose) { s_console ("I: send request to '%s' service:", service); zmsg_dump (request); } zmsg_send (&request, self->client); return 0; }
static void s_worker_send ( broker_t *self, worker_t *worker, char *command, char *option, zmsg_t *msg) { msg = msg? zmsg_dup (msg): zmsg_new (NULL); // Stack protocol envelope to start of message if (option) // Optional frame after command zmsg_push (msg, option); zmsg_push (msg, command); zmsg_push (msg, MDPW_WORKER); // Stack routing envelope to start of message zmsg_wrap (msg, worker->identity, ""); if (self->verbose) { s_console ("I: sending %s to worker", mdps_commands [(int) *command]); zmsg_dump (msg); } zmsg_send (&msg, self->socket); }
void worker_send (worker *worker, char *command, std::string option, zmsg *msg) { msg = (msg ? new zmsg(*msg) : new zmsg ()); // Stack protocol envelope to start of message if (option.size()>0) { // Optional frame after command msg->push_front ((char*)option.c_str()); } msg->push_front (command); msg->push_front ((char*)MDPW_WORKER); // Stack routing envelope to start of message msg->wrap(worker->m_identity.c_str(), ""); if (m_verbose) { s_console ("I: sending %s to worker", mdps_commands [(int) *command]); msg->dump (); } msg->send (*m_socket); delete msg; }
void s_broker_bind (broker_t *self, char *endpoint) { zmq_bind (self->socket, endpoint); s_console ("I: MDP broker/0.1.1 is active at %s", endpoint); }
zmsg * send (std::string service, zmsg *&request_p) { assert (request_p); zmsg *request = request_p; // Prefix request with protocol frames // Frame 1: "MDPCxy" (six bytes, MDP/Client x.y) // Frame 2: Service name (printable string) request->push_front((char*)service.c_str()); request->push_front((char*)MDPC_CLIENT); if (m_verbose) { s_console ("I: send request to '%s' service:", service.c_str()); request->dump(); } int retries_left = m_retries; while (retries_left && !s_interrupted) { zmsg * msg = new zmsg(*request); msg->send(*m_client); while (!s_interrupted) { // Poll socket for a reply, with timeout zmq::pollitem_t items [] = { { *m_client, 0, ZMQ_POLLIN, 0 } }; zmq::poll (items, 1, m_timeout); // If we got a reply, process it if (items [0].revents & ZMQ_POLLIN) { zmsg * recv_msg = new zmsg(*m_client); if (m_verbose) { s_console ("I: received reply:"); recv_msg->dump (); } // Don't try to handle errors, just assert noisily assert (recv_msg->parts () >= 3); std::basic_string<unsigned char> header = recv_msg->pop_front(); assert (header.compare((unsigned char *)MDPC_CLIENT) == 0); std::basic_string<unsigned char> reply_service = recv_msg->pop_front(); assert (reply_service.compare((unsigned char *)service.c_str()) == 0); delete request; return recv_msg; // Success } else { if (--retries_left) { if (m_verbose) { s_console ("W: no reply, reconnecting..."); } // Reconnect, and resend message connect_to_broker (); zmsg msg (*request); msg.send (*m_client); } else { if (m_verbose) { s_console ("W: permanent error, abandoning request"); } break; // Give up } } } } if (s_interrupted) { std::cout << "W: interrupt received, killing client..." << std::endl; } delete request; return 0; }
zmsg * recv (zmsg *&reply_p) { // Format and send the reply if we were provided one zmsg *reply = reply_p; assert (reply || !m_expect_reply); if (reply) { assert (m_reply_to.size()!=0); reply->wrap (m_reply_to.c_str(), ""); m_reply_to = ""; send_to_broker ((char*)MDPW_REPLY, "", reply); delete reply_p; reply_p = 0; } m_expect_reply = true; while (!s_interrupted) { zmq::pollitem_t items [] = { { *m_worker, 0, ZMQ_POLLIN, 0 } }; zmq::poll (items, 1, m_heartbeat * 1000); if (items [0].revents & ZMQ_POLLIN) { zmsg *msg = new zmsg(*m_worker); if (m_verbose) { s_console ("I: received message from broker:"); msg->dump (); } m_liveness = HEARTBEAT_LIVENESS; // Don't try to handle errors, just assert noisily assert (msg->parts () >= 3); std::basic_string<unsigned char> empty = msg->pop_front (); assert (empty.compare((unsigned char *)"") == 0); //assert (strcmp (empty, "") == 0); //free (empty); std::basic_string<unsigned char> header = msg->pop_front (); assert (header.compare((unsigned char *)MDPW_WORKER) == 0); //free (header); std::string command = (char*) msg->pop_front ().c_str(); if (command.compare (MDPW_REQUEST) == 0) { // We should pop and save as many addresses as there are // up to a null part, but for now, just save one... m_reply_to = msg->unwrap (); return msg; // We have a request to process } else if (command.compare (MDPW_HEARTBEAT) == 0) { // Do nothing for heartbeats } else if (command.compare (MDPW_DISCONNECT) == 0) { connect_to_broker (); } else { s_console ("E: invalid input message (%d)", (int) *(command.c_str())); msg->dump (); } delete msg; } else if (--m_liveness == 0) { if (m_verbose) { s_console ("W: disconnected from broker - retrying..."); } s_sleep (m_reconnect); connect_to_broker (); } // Send HEARTBEAT if it's time if (s_clock () > m_heartbeat_at) { send_to_broker ((char*)MDPW_HEARTBEAT, "", NULL); m_heartbeat_at = s_clock () + m_heartbeat; } } if (s_interrupted) printf ("W: interrupt received, killing worker...\n"); return NULL; }
zmsg_t * mdcli_send (mdcli_t *self, char *service, zmsg_t **request_p) { assert (self); assert (request_p); zmsg_t *request = *request_p; // Prefix request with protocol frames // Frame 1: "MDPCxy" (six bytes, MDP/Client x.y) // Frame 2: Service name (printable string) zmsg_push (request, service); zmsg_push (request, MDPC_CLIENT); if (self->verbose) { s_console ("I: send request to '%s' service:", service); zmsg_dump (request); } int retries_left = self->retries; while (retries_left && !s_interrupted) { zmsg_t *msg = zmsg_dup (request); zmsg_send (&msg, self->client); while (!s_interrupted) { // Poll socket for a reply, with timeout zmq_pollitem_t items [] = { { self->client, 0, ZMQ_POLLIN, 0 } }; zmq_poll (items, 1, self->timeout * 1000); // If we got a reply, process it if (items [0].revents & ZMQ_POLLIN) { zmsg_t *msg = zmsg_recv (self->client); if (self->verbose) { s_console ("I: received reply:"); zmsg_dump (msg); } // Don't try to handle errors, just assert noisily assert (zmsg_parts (msg) >= 3); char *header = zmsg_pop (msg); assert (streq (header, MDPC_CLIENT)); free (header); char *reply_service = zmsg_pop (msg); assert (streq (reply_service, service)); free (reply_service); zmsg_destroy (&request); return msg; // Success } else if (--retries_left) { if (self->verbose) s_console ("W: no reply, reconnecting..."); // Reconnect, and resend message s_mdcli_connect_to_broker (self); zmsg_t *msg = zmsg_dup (request); zmsg_send (&msg, self->client); } else { if (self->verbose) s_console ("W: permanent error, abandoning request"); break; // Give up } } } if (s_interrupted) printf ("W: interrupt received, killing client...\n"); zmsg_destroy (&request); return NULL; }