static void * client_task (void *args) { zctx_t *ctx = zctx_new (); void *client = zsocket_new (ctx, ZMQ_DEALER); // Set random identity to make tracing easier char identity [10]; sprintf (identity, "%04X-%04X", randof (0x10000), randof (0x10000)); zsockopt_set_identity (client, identity); zsocket_connect (client, "tcp://localhost:5570"); zmq_pollitem_t items [] = { { client, 0, ZMQ_POLLIN, 0 } }; int request_nbr = 0; while (true) { // Tick once per second, pulling in arriving messages int centitick; for (centitick = 0; centitick < 100; centitick++) { zmq_poll (items, 1, 10 * ZMQ_POLL_MSEC); if (items [0].revents & ZMQ_POLLIN) { zmsg_t *msg = zmsg_recv (client); zframe_print (zmsg_last (msg), identity); zmsg_destroy (&msg); } } zstr_send (client, "request #%d", ++request_nbr); } zctx_destroy (&ctx); return NULL; }
kosmonaut_t* kosmonaut_new(const char* addr, const char *vhost, const char *secret) { kosmonaut_t* self; uuid_t uuid; int addr_len = strlen(addr); char* sid = (char*)malloc(UUID_LEN * sizeof(char)); char identity[strlen(addr) + strlen(vhost) + strlen(secret) + 3 + 3]; self = (kosmonaut_t*)malloc(sizeof(kosmonaut_t)); if (!self) return NULL; pthread_mutex_init(&self->lmtx, NULL); pthread_mutex_init(&self->tmtx, NULL); self->running = 0; // not running, ready to run self->addr = (char*)malloc(addr_len * sizeof(char)); memcpy(self->addr, addr, addr_len * sizeof(char)); uuid_generate(uuid); uuid_unparse_lower(uuid, sid); self->ctx = zctx_new(); self->req = zsocket_new(self->ctx, ZMQ_REQ); sprintf(identity, "req:%s:%s:%s", vhost, secret, sid); zsockopt_set_identity(self->req, identity); self->dlr = zsocket_new(self->ctx, ZMQ_DEALER); sprintf(identity, "dlr:%s:%s:%s", vhost, secret, sid); zsockopt_set_identity(self->req, identity); free(sid); // TODO: register signal handlers return self; }
int main (int argc, char *argv []) { // First argument is this broker's name // Other arguments are our peers' names // if (argc < 2) { printf ("syntax: peering3 me {you}...\n"); exit (EXIT_FAILURE); } self = argv [1]; printf ("I: preparing broker at %s...\n", self); srandom ((unsigned) time (NULL)); // Prepare our context and sockets zctx_t *ctx = zctx_new (); char endpoint [256]; // Bind cloud frontend to endpoint void *cloudfe = zsocket_new (ctx, ZMQ_ROUTER); zsockopt_set_identity (cloudfe, self); zsocket_bind (cloudfe, "ipc://%s-cloud.ipc", self); // Bind state backend / publisher to endpoint void *statebe = zsocket_new (ctx, ZMQ_PUB); zsocket_bind (statebe, "ipc://%s-state.ipc", self); // Connect cloud backend to all peers void *cloudbe = zsocket_new (ctx, ZMQ_ROUTER); zsockopt_set_identity (cloudbe, self); int argn; for (argn = 2; argn < argc; argn++) { char *peer = argv [argn]; printf ("I: connecting to cloud frontend at '%s'\n", peer); zsocket_connect (cloudbe, "ipc://%s-cloud.ipc", peer); } // Connect statefe to all peers void *statefe = zsocket_new (ctx, ZMQ_SUB); for (argn = 2; argn < argc; argn++) { char *peer = argv [argn]; printf ("I: connecting to state backend at '%s'\n", peer); zsocket_connect (statefe, "ipc://%s-state.ipc", peer); } // Prepare local frontend and backend void *localfe = zsocket_new (ctx, ZMQ_ROUTER); zsocket_bind (localfe, "ipc://%s-localfe.ipc", self); void *localbe = zsocket_new (ctx, ZMQ_ROUTER); zsocket_bind (localbe, "ipc://%s-localbe.ipc", self); // Prepare monitor socket void *monitor = zsocket_new (ctx, ZMQ_PULL); zsocket_bind (monitor, "ipc://%s-monitor.ipc", self); // Start local workers int worker_nbr; for (worker_nbr = 0; worker_nbr < NBR_WORKERS; worker_nbr++) zthread_new (ctx, worker_task, NULL); // Start local clients int client_nbr; for (client_nbr = 0; client_nbr < NBR_CLIENTS; client_nbr++) zthread_new (ctx, client_task, NULL); // Interesting part // ------------------------------------------------------------- // Publish-subscribe flow // - Poll statefe and process capacity updates // - Each time capacity changes, broadcast new value // Request-reply flow // - Poll primary and process local/cloud replies // - While worker available, route localfe to local or cloud // Queue of available workers int local_capacity = 0; int cloud_capacity = 0; zlist_t *workers = zlist_new (); while (1) { zmq_pollitem_t primary [] = { { localbe, 0, ZMQ_POLLIN, 0 }, { cloudbe, 0, ZMQ_POLLIN, 0 }, { statefe, 0, ZMQ_POLLIN, 0 }, { monitor, 0, ZMQ_POLLIN, 0 } }; // If we have no workers anyhow, wait indefinitely int rc = zmq_poll (primary, 4, local_capacity? 1000 * ZMQ_POLL_MSEC: -1); if (rc == -1) break; // Interrupted // Track if capacity changes during this iteration int previous = local_capacity; // Handle reply from local worker zmsg_t *msg = NULL; if (primary [0].revents & ZMQ_POLLIN) { msg = zmsg_recv (localbe); if (!msg) break; // Interrupted zframe_t *address = zmsg_unwrap (msg); zlist_append (workers, address); local_capacity++; // If it's READY, don't route the message any further zframe_t *frame = zmsg_first (msg); if (memcmp (zframe_data (frame), LRU_READY, 1) == 0) zmsg_destroy (&msg); } // Or handle reply from peer broker else if (primary [1].revents & ZMQ_POLLIN) { msg = zmsg_recv (cloudbe); if (!msg) break; // Interrupted // We don't use peer broker address for anything zframe_t *address = zmsg_unwrap (msg); zframe_destroy (&address); } // Route reply to cloud if it's addressed to a broker for (argn = 2; msg && argn < argc; argn++) { char *data = (char *) zframe_data (zmsg_first (msg)); size_t size = zframe_size (zmsg_first (msg)); if (size == strlen (argv [argn]) && memcmp (data, argv [argn], size) == 0) zmsg_send (&msg, cloudfe); } // Route reply to client if we still need to if (msg) zmsg_send (&msg, localfe); // Handle capacity updates if (primary [2].revents & ZMQ_POLLIN) { char *status = zstr_recv (statefe); cloud_capacity = atoi (status); free (status); } // Handle monitor message if (primary [3].revents & ZMQ_POLLIN) { char *status = zstr_recv (monitor); printf ("%s\n", status); free (status); } // Now route as many clients requests as we can handle // - If we have local capacity we poll both localfe and cloudfe // - If we have cloud capacity only, we poll just localfe // - Route any request locally if we can, else to cloud // while (local_capacity + cloud_capacity) { zmq_pollitem_t secondary [] = { { localfe, 0, ZMQ_POLLIN, 0 }, { cloudfe, 0, ZMQ_POLLIN, 0 } }; if (local_capacity) rc = zmq_poll (secondary, 2, 0); else rc = zmq_poll (secondary, 1, 0); assert (rc >= 0); if (secondary [0].revents & ZMQ_POLLIN) msg = zmsg_recv (localfe); else if (secondary [1].revents & ZMQ_POLLIN) msg = zmsg_recv (cloudfe); else break; // No work, go back to primary if (local_capacity) { zframe_t *frame = (zframe_t *) zlist_pop (workers); zmsg_wrap (msg, frame); zmsg_send (&msg, localbe); local_capacity--; } else { // Route to random broker peer int random_peer = randof (argc - 2) + 2; zmsg_pushmem (msg, argv [random_peer], strlen (argv [random_peer])); zmsg_send (&msg, cloudbe); } } if (local_capacity != previous) { // We stick our own address onto the envelope zstr_sendm (statebe, self); // Broadcast new capacity zstr_sendf (statebe, "%d", local_capacity); } } // When we're done, clean up properly while (zlist_size (workers)) { zframe_t *frame = (zframe_t *) zlist_pop (workers); zframe_destroy (&frame); } zlist_destroy (&workers); zctx_destroy (&ctx); return EXIT_SUCCESS; }
int main(int argc, char* argv[]) { zctx_t* ctx; void* cloudfe; void* cloudbe; int i; void* localfe; void* localbe; int capacity = 0; zlist_t* workers; if (argc < 2) { fprintf(stderr, "syntax: peering me {you} ...\n"); return 1; } self = argv[1]; fprintf(stdout, "I: preparing broker at %s ...\n", self); srand((unsigned int)time(NULL)); ctx = zctx_new(); cloudfe = zsocket_new(ctx, ZMQ_ROUTER); zsockopt_set_identity(cloudfe, self); zsocket_bind(cloudfe, "ipc://%s-cloud.ipc", self); cloudbe = zsocket_new(ctx, ZMQ_ROUTER); zsockopt_set_identity(cloudbe, self); for (i = 2; i < argc; ++i) { char* peer = argv[i]; fprintf(stdout, "I: connecting to cloud frontend at '%s'\n", peer); zsocket_connect(cloudbe, "ipc://%s-cloud.ipc", peer); } localfe = zsocket_new(ctx, ZMQ_ROUTER); zsocket_bind(localfe, "ipc://%s-localfe.ipc", self); localbe = zsocket_new(ctx, ZMQ_ROUTER); zsocket_bind(localbe, "ipc://%s-localbe.ipc", self); fprintf(stdout, "Press Enter when all brokers are started: "); getchar(); for (i = 0; i < NUM_WORKERS; ++i) zthread_new(worker_routine, NULL); for (i = 0; i < NUM_CLIENTS; ++i) zthread_new(client_routine, NULL); workers = zlist_new(); while (1) { zmsg_t* msg; zmq_pollitem_t backends[] = { {localbe, 0, ZMQ_POLLIN, 0}, {cloudbe, 0, ZMQ_POLLIN, 0}, }; int r = zmq_poll(backends, 2, capacity ? 1000 * ZMQ_POLL_MSEC : -1); if (-1 == r) break; msg = NULL; if (backends[0].revents & ZMQ_POLLIN) { zframe_t* identity; zframe_t* frame; msg = zmsg_recv(localbe); if (!msg) break; identity = zmsg_unwrap(msg); zlist_append(workers, identity); ++capacity; frame = zmsg_first(msg); if (0 == memcmp(zframe_data(frame), WORKER_READY, 1)) zmsg_destroy(&msg); } else if (backends[1].revents & ZMQ_POLLIN) { zframe_t* identity; msg = zmsg_recv(cloudbe); if (!msg) break; identity = zmsg_unwrap(msg); zframe_destroy(&identity); } for (i = 2; msg && i < argc; ++i) { char* data = (char*)zframe_data(zmsg_first(msg)); size_t size = zframe_size(zmsg_first(msg)); if (size == strlen(argv[i]) && 0 == memcmp(data, argv[i], size)) zmsg_send(&msg, cloudfe); } if (msg) zmsg_send(&msg, localfe); while (capacity) { int reroutable = 0; zmq_pollitem_t frontends[] = { {localfe, 0, ZMQ_POLLIN, 0}, {cloudfe, 0, ZMQ_POLLIN, 0}, }; r = zmq_poll(frontends, 2, 0); assert(r >= 0); if (frontends[1].revents & ZMQ_POLLIN) { msg = zmsg_recv(cloudfe); reroutable = 0; } else if (frontends[0].revents & ZMQ_POLLIN) { msg = zmsg_recv(localfe); reroutable = 1; } else break; if (0 != reroutable && argc > 2 && 0 == rand() % 5) { int random_peer = rand() % (argc - 2) + 2; zmsg_pushmem(msg, argv[random_peer], strlen(argv[random_peer])); zmsg_send(&msg, cloudbe); } else { zframe_t* frame = (zframe_t*)zlist_pop(workers); zmsg_wrap(msg, frame); zmsg_send(&msg, localbe); --capacity; } } } while (zlist_size(workers)) { zframe_t* frame = (zframe_t*)zlist_pop(workers); zframe_destroy(&frame); } zlist_destroy(&workers); zctx_destroy(&ctx); return 0; }
int main (int argc, char *argv []) { // First argument is this broker's name // Other arguments are our peers' names // if (argc < 2) { printf ("syntax: peering2 me {you}...\n"); exit (EXIT_FAILURE); } self = argv [1]; printf ("I: preparing broker at %s...\n", self); srandom ((unsigned) time (NULL)); zctx_t *ctx = zctx_new (); // Bind cloud frontend to endpoint void *cloudfe = zsocket_new (ctx, ZMQ_ROUTER); zsockopt_set_identity (cloudfe, self); zsocket_bind (cloudfe, "ipc://%s-cloud.ipc", self); // Connect cloud backend to all peers void *cloudbe = zsocket_new (ctx, ZMQ_ROUTER); zsockopt_set_identity (cloudbe, self); int argn; for (argn = 2; argn < argc; argn++) { char *peer = argv [argn]; printf ("I: connecting to cloud frontend at '%s'\n", peer); zsocket_connect (cloudbe, "ipc://%s-cloud.ipc", peer); } // Prepare local frontend and backend void *localfe = zsocket_new (ctx, ZMQ_ROUTER); zsocket_bind (localfe, "ipc://%s-localfe.ipc", self); void *localbe = zsocket_new (ctx, ZMQ_ROUTER); zsocket_bind (localbe, "ipc://%s-localbe.ipc", self); // Get user to tell us when we can start... printf ("Press Enter when all brokers are started: "); getchar (); // Start local workers int worker_nbr; for (worker_nbr = 0; worker_nbr < NBR_WORKERS; worker_nbr++) zthread_new (worker_task, NULL); // Start local clients int client_nbr; for (client_nbr = 0; client_nbr < NBR_CLIENTS; client_nbr++) zthread_new (client_task, NULL); // .split request-reply handling // Here we handle the request-reply flow. We're using the LRU approach // to poll workers at all times, and clients only when there are one or // more workers available. // Least recently used queue of available workers int capacity = 0; zlist_t *workers = zlist_new (); while (true) { // First, route any waiting replies from workers zmq_pollitem_t backends [] = { { localbe, 0, ZMQ_POLLIN, 0 }, { cloudbe, 0, ZMQ_POLLIN, 0 } }; // If we have no workers anyhow, wait indefinitely int rc = zmq_poll (backends, 2, capacity? 1000 * ZMQ_POLL_MSEC: -1); if (rc == -1) break; // Interrupted // Handle reply from local worker zmsg_t *msg = NULL; if (backends [0].revents & ZMQ_POLLIN) { msg = zmsg_recv (localbe); if (!msg) break; // Interrupted zframe_t *address = zmsg_unwrap (msg); zlist_append (workers, address); capacity++; // If it's READY, don't route the message any further zframe_t *frame = zmsg_first (msg); if (memcmp (zframe_data (frame), LRU_READY, 1) == 0) zmsg_destroy (&msg); } // Or handle reply from peer broker else if (backends [1].revents & ZMQ_POLLIN) { msg = zmsg_recv (cloudbe); if (!msg) break; // Interrupted // We don't use peer broker address for anything zframe_t *address = zmsg_unwrap (msg); zframe_destroy (&address); } // Route reply to cloud if it's addressed to a broker for (argn = 2; msg && argn < argc; argn++) { char *data = (char *) zframe_data (zmsg_first (msg)); size_t size = zframe_size (zmsg_first (msg)); if (size == strlen (argv [argn]) && memcmp (data, argv [argn], size) == 0) zmsg_send (&msg, cloudfe); } // Route reply to client if we still need to if (msg) zmsg_send (&msg, localfe); // .split route client requests // Now we route as many client requests as we have worker capacity // for. We may reroute requests from our local frontend, but not from // the cloud frontend. We reroute randomly now, just to test things // out. In the next version we'll do this properly by calculating // cloud capacity: while (capacity) { zmq_pollitem_t frontends [] = { { localfe, 0, ZMQ_POLLIN, 0 }, { cloudfe, 0, ZMQ_POLLIN, 0 } }; rc = zmq_poll (frontends, 2, 0); assert (rc >= 0); int reroutable = 0; // We'll do peer brokers first, to prevent starvation if (frontends [1].revents & ZMQ_POLLIN) { msg = zmsg_recv (cloudfe); reroutable = 0; } else if (frontends [0].revents & ZMQ_POLLIN) { msg = zmsg_recv (localfe); reroutable = 1; } else break; // No work, go back to backends // If reroutable, send to cloud 20% of the time // Here we'd normally use cloud status information // if (reroutable && argc > 2 && randof (5) == 0) { // Route to random broker peer int random_peer = randof (argc - 2) + 2; zmsg_pushmem (msg, argv [random_peer], strlen (argv [random_peer])); zmsg_send (&msg, cloudbe); } else { zframe_t *frame = (zframe_t *) zlist_pop (workers); zmsg_wrap (msg, frame); zmsg_send (&msg, localbe); capacity--; } } } // When we're done, clean up properly while (zlist_size (workers)) { zframe_t *frame = (zframe_t *) zlist_pop (workers); zframe_destroy (&frame); } zlist_destroy (&workers); zctx_destroy (&ctx); return EXIT_SUCCESS; }
int main (int argc, char *argv []) { // First argument is this broker's name // Other arguments are our peers' names // if (argc < 2) { printf ("syntax: peering3 me {you}...\n"); exit (EXIT_FAILURE); } self = argv [1]; printf ("I: preparing broker at %s...\n", self); srandom ((unsigned) time (NULL)); zctx_t *ctx = zctx_new (); // Prepare local frontend and backend void *localfe = zsocket_new (ctx, ZMQ_ROUTER); zsocket_bind (localfe, "ipc://%s-localfe.ipc", self); void *localbe = zsocket_new (ctx, ZMQ_ROUTER); zsocket_bind (localbe, "ipc://%s-localbe.ipc", self); // Bind cloud frontend to endpoint void *cloudfe = zsocket_new (ctx, ZMQ_ROUTER); zsockopt_set_identity (cloudfe, self); zsocket_bind (cloudfe, "ipc://%s-cloud.ipc", self); // Connect cloud backend to all peers void *cloudbe = zsocket_new (ctx, ZMQ_ROUTER); zsockopt_set_identity (cloudbe, self); int argn; for (argn = 2; argn < argc; argn++) { char *peer = argv [argn]; printf ("I: connecting to cloud frontend at '%s'\n", peer); zsocket_connect (cloudbe, "ipc://%s-cloud.ipc", peer); } // Bind state backend to endpoint void *statebe = zsocket_new (ctx, ZMQ_PUB); zsocket_bind (statebe, "ipc://%s-state.ipc", self); // Connect state frontend to all peers void *statefe = zsocket_new (ctx, ZMQ_SUB); zsockopt_set_subscribe (statefe, ""); for (argn = 2; argn < argc; argn++) { char *peer = argv [argn]; printf ("I: connecting to state backend at '%s'\n", peer); zsocket_connect (statefe, "ipc://%s-state.ipc", peer); } // Prepare monitor socket void *monitor = zsocket_new (ctx, ZMQ_PULL); zsocket_bind (monitor, "ipc://%s-monitor.ipc", self); // .split start child tasks // After binding and connecting all our sockets, we start our child // tasks - workers and clients: int worker_nbr; for (worker_nbr = 0; worker_nbr < NBR_WORKERS; worker_nbr++) zthread_new (worker_task, NULL); // Start local clients int client_nbr; for (client_nbr = 0; client_nbr < NBR_CLIENTS; client_nbr++) zthread_new (client_task, NULL); // Queue of available workers int local_capacity = 0; int cloud_capacity = 0; zlist_t *workers = zlist_new (); // .split main loop // The main loop has two parts. First we poll workers and our two service // sockets (statefe and monitor), in any case. If we have no ready workers, // there's no point in looking at incoming requests. These can remain on // their internal 0MQ queues: while (true) { zmq_pollitem_t primary [] = { { localbe, 0, ZMQ_POLLIN, 0 }, { cloudbe, 0, ZMQ_POLLIN, 0 }, { statefe, 0, ZMQ_POLLIN, 0 }, { monitor, 0, ZMQ_POLLIN, 0 } }; // If we have no workers ready, wait indefinitely int rc = zmq_poll (primary, 4, local_capacity? 1000 * ZMQ_POLL_MSEC: -1); if (rc == -1) break; // Interrupted // Track if capacity changes during this iteration int previous = local_capacity; // Handle reply from local worker zmsg_t *msg = NULL; if (primary [0].revents & ZMQ_POLLIN) { msg = zmsg_recv (localbe); if (!msg) break; // Interrupted zframe_t *identity = zmsg_unwrap (msg); zlist_append (workers, identity); local_capacity++; // If it's READY, don't route the message any further zframe_t *frame = zmsg_first (msg); if (memcmp (zframe_data (frame), WORKER_READY, 1) == 0) zmsg_destroy (&msg); } // Or handle reply from peer broker else if (primary [1].revents & ZMQ_POLLIN) { msg = zmsg_recv (cloudbe); if (!msg) break; // Interrupted // We don't use peer broker identity for anything zframe_t *identity = zmsg_unwrap (msg); zframe_destroy (&identity); } // Route reply to cloud if it's addressed to a broker for (argn = 2; msg && argn < argc; argn++) { char *data = (char *) zframe_data (zmsg_first (msg)); size_t size = zframe_size (zmsg_first (msg)); if (size == strlen (argv [argn]) && memcmp (data, argv [argn], size) == 0) zmsg_send (&msg, cloudfe); } // Route reply to client if we still need to if (msg) zmsg_send (&msg, localfe); // .split handle state messages // If we have input messages on our statefe or monitor sockets we // can process these immediately: if (primary [2].revents & ZMQ_POLLIN) { char *peer = zstr_recv (statefe); char *status = zstr_recv (statefe); cloud_capacity = atoi (status); free (peer); free (status); } if (primary [3].revents & ZMQ_POLLIN) { char *status = zstr_recv (monitor); printf ("%s\n", status); free (status); } // .split route client requests // Now route as many clients requests as we can handle. If we have // local capacity we poll both localfe and cloudfe. If we have cloud // capacity only, we poll just localfe. We route any request locally // if we can, else we route to the cloud. while (local_capacity + cloud_capacity) { zmq_pollitem_t secondary [] = { { localfe, 0, ZMQ_POLLIN, 0 }, { cloudfe, 0, ZMQ_POLLIN, 0 } }; if (local_capacity) rc = zmq_poll (secondary, 2, 0); else rc = zmq_poll (secondary, 1, 0); assert (rc >= 0); if (secondary [0].revents & ZMQ_POLLIN) msg = zmsg_recv (localfe); else if (secondary [1].revents & ZMQ_POLLIN) msg = zmsg_recv (cloudfe); else break; // No work, go back to primary if (local_capacity) { zframe_t *frame = (zframe_t *) zlist_pop (workers); zmsg_wrap (msg, frame); zmsg_send (&msg, localbe); local_capacity--; } else { // Route to random broker peer int random_peer = randof (argc - 2) + 2; zmsg_pushmem (msg, argv [random_peer], strlen (argv [random_peer])); zmsg_send (&msg, cloudbe); } } // .split broadcast capacity // We broadcast capacity messages to other peers; to reduce chatter // we do this only if our capacity changed. if (local_capacity != previous) { // We stick our own identity onto the envelope zstr_sendm (statebe, self); // Broadcast new capacity zstr_send (statebe, "%d", local_capacity); } } // When we're done, clean up properly while (zlist_size (workers)) { zframe_t *frame = (zframe_t *) zlist_pop (workers); zframe_destroy (&frame); } zlist_destroy (&workers); zctx_destroy (&ctx); return EXIT_SUCCESS; }