static void
server_process_cluster_command (
server_t *self,
const char *peer_id,
const char *peer_name,
zmsg_t *msg,
bool unicast)
{
char *request = zmsg_popstr (msg);
char *pipename = zmsg_popstr (msg);
zsys_info ("peer=%s command=%s pipe=%s unicast=%d",
peer_name, request, pipename? pipename: "-", unicast);
// Lookup or create pipe
// TODO: remote pipes need cleaning up with some timeout
pipe_t *pipe = NULL;
if (pipename) {
pipe = (pipe_t *) zhash_lookup (self->pipes, pipename);
if (!pipe)
pipe = pipe_new (self, pipename);
}
if (pipe && streq (request, "HAVE WRITER"))
pipe_attach_remote_writer (pipe, peer_id, unicast);
else
if (pipe && streq (request, "HAVE READER"))
pipe_attach_remote_reader (pipe, peer_id, unicast);
else
if (pipe && streq (request, "DATA")) {
// TODO encode these commands as proper protocol
zframe_t *frame = zmsg_pop (msg);
zchunk_t *chunk = zchunk_new (zframe_data (frame), zframe_size (frame));
if (pipe->writer == REMOTE_NODE && pipe->reader) {
zsys_info ("send %d bytes to pipe", (int) zchunk_size (chunk));
pipe_send_data (pipe, &chunk);
}
else
zsys_info ("discard %d bytes, unroutable", (int) zchunk_size (chunk));
zframe_destroy (&frame);
zchunk_destroy (&chunk);
}
else
if (pipe && streq (request, "DROP READER"))
pipe_drop_remote_reader (&pipe, peer_id);
else
if (pipe && streq (request, "DROP WRITER"))
pipe_drop_remote_writer (&pipe, peer_id);
else
if (streq (request, "DUMP"))
zyre_dump (self->zyre);
else
zsys_warning ("bad request %s from %s", request, peer_name);
zstr_free (&pipename);
zstr_free (&request);
}
void
zyre_test (bool verbose)
{
printf (" * zyre: ");
if (verbose)
printf ("\n");
// @selftest
// We'll use inproc gossip discovery so that this works without networking
uint64_t version = zyre_version ();
assert ((version / 10000) % 100 == ZYRE_VERSION_MAJOR);
assert ((version / 100) % 100 == ZYRE_VERSION_MINOR);
assert (version % 100 == ZYRE_VERSION_PATCH);
// Create two nodes
zyre_t *node1 = zyre_new ("node1");
assert (node1);
assert (streq (zyre_name (node1), "node1"));
zyre_set_header (node1, "X-HELLO", "World");
if (verbose)
zyre_set_verbose (node1);
// Set inproc endpoint for this node
int rc = zyre_set_endpoint (node1, "inproc://zyre-node1");
assert (rc == 0);
// Set up gossip network for this node
zyre_gossip_bind (node1, "inproc://gossip-hub");
rc = zyre_start (node1);
assert (rc == 0);
zyre_t *node2 = zyre_new ("node2");
assert (node2);
assert (streq (zyre_name (node2), "node2"));
if (verbose)
zyre_set_verbose (node2);
// Set inproc endpoint for this node
// First, try to use existing name, it'll fail
rc = zyre_set_endpoint (node2, "inproc://zyre-node1");
assert (rc == -1);
// Now use available name and confirm that it succeeds
rc = zyre_set_endpoint (node2, "inproc://zyre-node2");
assert (rc == 0);
// Set up gossip network for this node
zyre_gossip_connect (node2, "inproc://gossip-hub");
rc = zyre_start (node2);
assert (rc == 0);
assert (strneq (zyre_uuid (node1), zyre_uuid (node2)));
zyre_join (node1, "GLOBAL");
zyre_join (node2, "GLOBAL");
// Give time for them to interconnect
zclock_sleep (250);
if (verbose)
zyre_dump (node1);
zlist_t *peers = zyre_peers (node1);
assert (peers);
assert (zlist_size (peers) == 1);
zlist_destroy (&peers);
zyre_join (node1, "node1 group of one");
zyre_join (node2, "node2 group of one");
// Give them time to join their groups
zclock_sleep (250);
zlist_t *own_groups = zyre_own_groups (node1);
assert (own_groups);
assert (zlist_size (own_groups) == 2);
zlist_destroy (&own_groups);
zlist_t *peer_groups = zyre_peer_groups (node1);
assert (peer_groups);
assert (zlist_size (peer_groups) == 2);
zlist_destroy (&peer_groups);
char *value = zyre_peer_header_value (node2, zyre_uuid (node1), "X-HELLO");
assert (streq (value, "World"));
zstr_free (&value);
// One node shouts to GLOBAL
zyre_shouts (node1, "GLOBAL", "Hello, World");
// Second node should receive ENTER, JOIN, and SHOUT
zmsg_t *msg = zyre_recv (node2);
assert (msg);
char *command = zmsg_popstr (msg);
assert (streq (command, "ENTER"));
zstr_free (&command);
assert (zmsg_size (msg) == 4);
char *peerid = zmsg_popstr (msg);
char *name = zmsg_popstr (msg);
assert (streq (name, "node1"));
zstr_free (&name);
zframe_t *headers_packed = zmsg_pop (msg);
char *address = zmsg_popstr (msg);
char *endpoint = zyre_peer_address (node2, peerid);
assert (streq (address, endpoint));
zstr_free (&peerid);
zstr_free (&endpoint);
zstr_free (&address);
assert (headers_packed);
zhash_t *headers = zhash_unpack (headers_packed);
assert (headers);
zframe_destroy (&headers_packed);
assert (streq ((char *) zhash_lookup (headers, "X-HELLO"), "World"));
zhash_destroy (&headers);
zmsg_destroy (&msg);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "JOIN"));
zstr_free (&command);
assert (zmsg_size (msg) == 3);
zmsg_destroy (&msg);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "JOIN"));
zstr_free (&command);
assert (zmsg_size (msg) == 3);
zmsg_destroy (&msg);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "SHOUT"));
zstr_free (&command);
zmsg_destroy (&msg);
zyre_stop (node2);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "STOP"));
zstr_free (&command);
zmsg_destroy (&msg);
zyre_stop (node1);
zyre_destroy (&node1);
zyre_destroy (&node2);
printf ("OK\n");
#ifdef ZYRE_BUILD_DRAFT_API
if (zsys_has_curve()){
printf (" * zyre-curve: ");
if (verbose)
printf ("\n");
if (verbose)
zsys_debug("----------------TESTING CURVE --------------");
zactor_t *speaker = zactor_new (zbeacon, NULL);
assert (speaker);
if (verbose)
zstr_sendx (speaker, "VERBOSE", NULL);
// ensuring we have a broadcast address
zsock_send (speaker, "si", "CONFIGURE", 9999);
char *hostname = zstr_recv (speaker);
if (!*hostname) {
printf ("OK (skipping test, no UDP broadcasting)\n");
zactor_destroy (&speaker);
freen (hostname);
return;
}
freen (hostname);
zactor_destroy (&speaker);
// zap setup
zactor_t *auth = zactor_new(zauth, NULL);
assert (auth);
if (verbose) {
zstr_sendx(auth, "VERBOSE", NULL);
zsock_wait(auth);
}
zstr_sendx (auth, "CURVE", CURVE_ALLOW_ANY, NULL);
zsock_wait (auth);
zyre_t *node3 = zyre_new ("node3");
zyre_t *node4 = zyre_new ("node4");
assert (node3);
assert (node4);
zyre_set_verbose (node3);
zyre_set_verbose (node4);
zyre_set_zap_domain(node3, "TEST");
zyre_set_zap_domain(node4, "TEST");
zsock_set_rcvtimeo(node3->inbox, 10000);
zsock_set_rcvtimeo(node4->inbox, 10000);
zcert_t *node3_cert = zcert_new ();
zcert_t *node4_cert = zcert_new ();
assert (node3_cert);
assert (node4_cert);
zyre_set_zcert(node3, node3_cert);
zyre_set_zcert(node4, node4_cert);
zyre_set_header(node3, "X-PUBLICKEY", "%s", zcert_public_txt(node3_cert));
zyre_set_header(node4, "X-PUBLICKEY", "%s", zcert_public_txt(node4_cert));
// test beacon
if (verbose)
zsys_debug ("----------------TESTING BEACON----------------");
rc = zyre_start(node3);
assert (rc == 0);
rc = zyre_start(node4);
assert (rc == 0);
zyre_join (node3, "GLOBAL");
zyre_join (node4, "GLOBAL");
zclock_sleep (1500);
if (verbose) {
zyre_dump (node3);
zyre_dump (node4);
}
zyre_shouts (node3, "GLOBAL", "Hello, World");
// Second node should receive ENTER, JOIN, and SHOUT
msg = zyre_recv (node4);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "ENTER"));
zstr_free (&command);
char *peerid = zmsg_popstr (msg);
assert (peerid);
name = zmsg_popstr (msg);
assert (streq (name, "node3"));
zmsg_destroy (&msg);
msg = zyre_recv (node4);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "JOIN"));
zstr_free (&command);
zmsg_destroy(&msg);
msg = zyre_recv (node4);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "SHOUT"));
zstr_free (&command);
zmsg_destroy(&msg);
zyre_leave(node3, "GLOBAL");
zyre_leave(node4, "GLOBAL");
zstr_free (&name);
zstr_free (&peerid);
zstr_free (&command);
zyre_stop (node3);
zyre_stop (node4);
// give things a chance to settle...
zclock_sleep (250);
zyre_destroy(&node3);
zyre_destroy(&node4);
zcert_destroy(&node3_cert);
zcert_destroy(&node4_cert);
// test gossip
if (verbose)
zsys_debug ("----------------TESTING GOSSIP----------------");
zyre_t *node5 = zyre_new ("node5");
zyre_t *node6 = zyre_new ("node6");
assert (node5);
assert (node6);
if (verbose) {
zyre_set_verbose (node5);
zyre_set_verbose (node6);
}
// if it takes more than 10s, something probably went terribly wrong
zsock_set_rcvtimeo(node5->inbox, 10000);
zsock_set_rcvtimeo(node6->inbox, 10000);
zcert_t *node5_cert = zcert_new ();
zcert_t *node6_cert = zcert_new ();
assert (node5_cert);
assert (node6_cert);
zyre_set_zcert(node5, node5_cert);
zyre_set_zcert(node6, node6_cert);
zyre_set_header(node5, "X-PUBLICKEY", "%s", zcert_public_txt(node5_cert));
zyre_set_header(node6, "X-PUBLICKEY", "%s", zcert_public_txt(node6_cert));
const char *gossip_cert = zcert_public_txt (node5_cert);
// TODO- need to add zyre_gossip_port functions to get port from gossip bind(?)
zyre_gossip_bind(node5, "tcp://127.0.0.1:9001");
zyre_gossip_connect_curve(node6, gossip_cert, "tcp://127.0.0.1:9001");
zyre_start(node5);
zsock_wait(node5);
zyre_start(node6);
zsock_wait(node6);
zyre_join (node5, "GLOBAL");
zyre_join (node6, "GLOBAL");
// give things a chance to settle...
zclock_sleep (1500);
if (verbose) {
zyre_dump (node5);
zyre_dump (node6);
}
zyre_shouts (node5, "GLOBAL", "Hello, World");
// Second node should receive ENTER, JOIN, and SHOUT
msg = zyre_recv (node6);
assert (msg);
command = zmsg_popstr (msg);
zsys_info(command);
assert (streq (command, "ENTER"));
zstr_free (&command);
peerid = zmsg_popstr (msg);
assert (peerid);
name = zmsg_popstr (msg);
zmsg_destroy (&msg);
assert (streq (name, "node5"));
zstr_free (&name);
zyre_leave(node5, "GLOBAL");
zyre_leave(node6, "GLOBAL");
zyre_stop (node5);
zyre_stop (node6);
// give things a chance to settle...
zclock_sleep (250);
zstr_free (&peerid);
zcert_destroy (&node5_cert);
zcert_destroy (&node6_cert);
zyre_destroy(&node5);
zyre_destroy(&node6);
zactor_destroy(&auth);
printf ("OK\n");
}
#endif
}
void
ztask_monitor_api_loop (ztask_monitor_api_t *self)
{
zyre_t *node = zyre_new();
zyre_set_header(node, "X-ZTASK-MON", "TASK MON");
zyre_start(node);
zyre_dump(node);
if (self->pub_port >= 0)
zclock_log("Publisher is listening on port %d ...", self->pub_port);
zpoller_t *poller = zpoller_new (zyre_socket (node), NULL);
uint64_t reap_at = zclock_time () + self->pub_timeout;
zyre_event_t *event;
while (!zsys_interrupted) {
void *which = zpoller_wait (poller, self->pub_timeout);
if (zpoller_terminated (poller)) {
break;
}
if (which == zyre_socket (node)) {
event = zyre_event_new(node);
if (!event)
break;
if (zyre_event_type(event) == ZYRE_EVENT_ENTER) {
}
else
if (zyre_event_type(event) == ZYRE_EVENT_EXIT) {
char *data = (char *) zhash_lookup(self->nodes,
(const char *) zyre_event_sender(event));
if (data) {
zhash_delete(self->nodes,
(const char *) zyre_event_sender(event));
free(data);
}
}
else
if (zyre_event_type(event) == ZYRE_EVENT_WHISPER) {
zmsg_t *msg = zyre_event_msg(event);
char *cpu_str = zmsg_popstr(msg);
char *mem_rss_str = zmsg_popstr(msg);
if (self->pub_sock) {
char *data = (char *) zhash_lookup(self->nodes,
(const char *) zyre_event_sender(event));
if (data) {
zhash_delete(self->nodes,
(const char *) zyre_event_sender(event));
free(data);
}
if (asprintf(&data, "{\"id\":\"%s\",%s,%s}",
zyre_event_sender(event), cpu_str, mem_rss_str) < 0)
data = NULL;
zhash_insert(self->nodes,
(const char *) zyre_event_sender(event), data);
}
else {
zclock_log("From %s", zyre_event_sender(event));
zclock_log(" %s", cpu_str);
zclock_log(" %s", mem_rss_str);
}
free(cpu_str);
free(mem_rss_str);
}
zyre_event_destroy(&event);
}
if (zclock_time() >= reap_at) {
reap_at = zclock_time() + self->pub_timeout;
if (self->pub_sock) {
char *data = ztask_monitor_api_json (self);
if (data) {
zmsg_t *msg = zmsg_new();
zmsg_addstr(msg, data);
zclock_log("Sending %s ...", data);
zmsg_send(&msg, self->pub_sock);
free(data);
}
}
}
}
zyre_stop(node);
zpoller_destroy (&poller);
zyre_destroy(&node);
// Cleanup of self->nodes
// char *wk = (char *) zhash_first (self->nodes);
// while (wk) {
// zhash_delete (self->nodes, wk);
//// free (data);
// wk = (char *) zhash_first (self->nodes);
// }
}
void
zyre_test (bool verbose)
{
printf (" * zyre: ");
if (verbose)
printf ("\n");
// @selftest
// We'll use inproc gossip discovery so that this works without networking
int major, minor, patch;
zyre_version (&major, &minor, &patch);
assert (major == ZYRE_VERSION_MAJOR);
assert (minor == ZYRE_VERSION_MINOR);
assert (patch == ZYRE_VERSION_PATCH);
// Create two nodes
zyre_t *node1 = zyre_new ("node1");
assert (node1);
assert (streq (zyre_name (node1), "node1"));
zyre_set_header (node1, "X-HELLO", "World");
if (verbose)
zyre_set_verbose (node1);
// Set inproc endpoint for this node
int rc = zyre_set_endpoint (node1, "inproc://zyre-node1");
assert (rc == 0);
// Set up gossip network for this node
zyre_gossip_bind (node1, "inproc://gossip-hub");
rc = zyre_start (node1);
assert (rc == 0);
zyre_t *node2 = zyre_new ("node2");
assert (node2);
assert (streq (zyre_name (node2), "node2"));
if (verbose)
zyre_set_verbose (node2);
// Set inproc endpoint for this node
// First, try to use existing name, it'll fail
rc = zyre_set_endpoint (node2, "inproc://zyre-node1");
assert (rc == -1);
// Now use available name and confirm that it succeeds
rc = zyre_set_endpoint (node2, "inproc://zyre-node2");
assert (rc == 0);
// Set up gossip network for this node
zyre_gossip_connect (node2, "inproc://gossip-hub");
rc = zyre_start (node2);
assert (rc == 0);
assert (strneq (zyre_uuid (node1), zyre_uuid (node2)));
zyre_join (node1, "GLOBAL");
zyre_join (node2, "GLOBAL");
// Give time for them to interconnect
zclock_sleep (250);
if (verbose)
zyre_dump (node1);
zlist_t *peers = zyre_peers (node1);
assert (peers);
assert (zlist_size (peers) == 1);
zlist_destroy (&peers);
zyre_join (node1, "node1 group of one");
zyre_join (node2, "node2 group of one");
// Give them time to join their groups
zclock_sleep (250);
zlist_t *own_groups = zyre_own_groups (node1);
assert (own_groups);
assert (zlist_size (own_groups) == 2);
zlist_destroy (&own_groups);
zlist_t *peer_groups = zyre_peer_groups (node1);
assert (peer_groups);
assert (zlist_size (peer_groups) == 2);
zlist_destroy (&peer_groups);
char *value = zyre_peer_header_value (node2, zyre_uuid (node1), "X-HELLO");
assert (streq (value, "World"));
zstr_free (&value);
// One node shouts to GLOBAL
zyre_shouts (node1, "GLOBAL", "Hello, World");
// Second node should receive ENTER, JOIN, and SHOUT
zmsg_t *msg = zyre_recv (node2);
assert (msg);
char *command = zmsg_popstr (msg);
assert (streq (command, "ENTER"));
zstr_free (&command);
assert (zmsg_size (msg) == 4);
char *peerid = zmsg_popstr (msg);
char *name = zmsg_popstr (msg);
assert (streq (name, "node1"));
zstr_free (&name);
zframe_t *headers_packed = zmsg_pop (msg);
char *address = zmsg_popstr (msg);
char *endpoint = zyre_peer_address (node2, peerid);
assert (streq (address, endpoint));
zstr_free (&peerid);
zstr_free (&endpoint);
zstr_free (&address);
assert (headers_packed);
zhash_t *headers = zhash_unpack (headers_packed);
assert (headers);
zframe_destroy (&headers_packed);
assert (streq ((char *) zhash_lookup (headers, "X-HELLO"), "World"));
zhash_destroy (&headers);
zmsg_destroy (&msg);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "JOIN"));
zstr_free (&command);
assert (zmsg_size (msg) == 3);
zmsg_destroy (&msg);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "JOIN"));
zstr_free (&command);
assert (zmsg_size (msg) == 3);
zmsg_destroy (&msg);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "SHOUT"));
zstr_free (&command);
zmsg_destroy (&msg);
zyre_stop (node2);
msg = zyre_recv (node2);
assert (msg);
command = zmsg_popstr (msg);
assert (streq (command, "STOP"));
zstr_free (&command);
zmsg_destroy (&msg);
zyre_stop (node1);
zyre_destroy (&node1);
zyre_destroy (&node2);
// @end
printf ("OK\n");
}
void
ztask_run_manager_loop (ztask_run_manager_t *self)
{
assert (self);
if (!self->packetizer) {
zclock_log ("E: No packetizer set !!!");
return;
}
// Setting network interface if neede
if (self->node_interface)
zyre_set_interface(self->node, self->node_interface);
zyre_set_header (self->node, "X-ZTASK-RUN", "ZTASK RUN");
zyre_start (self->node);
zyre_dump (self->node);
zclock_sleep (10);
// ztask_packetizer_dump (self->packetizer);
ztask_packet_t *packet;
int request_sent = 0;
zyre_event_t *event;
while (!zsys_interrupted) {
if (ztask_packetizer_is_finished (self->packetizer))
break;
event = zyre_event_new (self->node);
if (!event)
break;
if (zyre_event_type (event) == ZYRE_EVENT_ENTER) {
// Ignoring nodes which don't have service X-ZTASK-NODE defined
if (zyre_event_header (event, "X-ZTASK-NODE")) {
zhash_insert (self->nodes, zyre_event_sender (event),"");
ztask_log_debug (self->log, "Adding node -> workers=%ld", zhash_size (self->nodes));
} else {
// TODO disconnect worker (zyre peer) somehow
}
}
else
if (zyre_event_type (event) == ZYRE_EVENT_EXIT) {
if (zhash_lookup (self->nodes, zyre_event_sender (event))) {
ztask_log_debug (self->log, "Removing node -> workers=%ld", zhash_size (self->nodes));
zhash_delete (self->nodes, zyre_event_sender (event));
// Cleanup packetizer in case ztask_node was killed
request_sent -= ztask_packetizer_running_node_cleanup (self->packetizer, zyre_event_sender (event));
}
}
else
if (zyre_event_type (event) == ZYRE_EVENT_WHISPER) {
// Ingoring whispers when they are not from our nodes
if (!zhash_lookup(self->nodes, zyre_event_sender (event))) {
ztask_log_warning (self->log, "W: Ingoring whisper from %s", zyre_event_sender (event));
zyre_event_destroy (&event);
continue;
}
zmsg_t *msg_report = zyre_event_msg(event);
char *str_msg = zmsg_popstr (msg_report);
if (streq(str_msg, "REQUEST")) {
// Let's handle request
packet = ztask_packetizer_next_packet (self->packetizer);
if (packet) {
char *cmd;
if ( asprintf (&cmd, "%s", ztask_packet_cmd (packet)) < 0)
cmd = NULL;
assert (cmd);
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "REQUEST");
zmsg_addstrf (msg, "%s_%ld", zyre_uuid (self->node), ztask_packet_id (packet));
zmsg_addstr (msg, cmd);
zyre_whisper (self->node, zyre_event_sender (event), &msg);
ztask_packet_set_node (packet, zyre_event_sender (event));
request_sent++;
ztask_packetizer_info (self->packetizer, request_sent);
ztask_log_debug (self->log, "ID=%s_%ld cmd='%s' running=%d", zyre_uuid (self->node), ztask_packet_id (packet), cmd, request_sent);
free (cmd);
}
else {
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "NO_PACKETS");
zyre_whisper (self->node, zyre_event_sender (event), &msg);
}
}
else
if (streq(str_msg, "REPORT")) {
// It's report
char *str_id = zmsg_popstr (msg_report);
char *str_pid_rc = zmsg_popstr (msg_report);
ztask_log_debug (self->log, "REPORT ID=%s rc=%s", str_id, str_pid_rc);
ztask_packetizer_report (self->packetizer, str_id, str_pid_rc);
request_sent--;
ztask_packetizer_info (self->packetizer, request_sent);
free (str_id);
free (str_pid_rc);
}
else {
ztask_log_error (self->log, "E: ztask_run_manager_loop : Wrong message %s", str_msg);
assert (false);
}
free (str_msg);
}
zyre_event_destroy (&event);
}
// FIXME : simplify zhash_foreach()
zlist_t *keys = zhash_keys (self->nodes);
char *key = (char *) zlist_first (keys);
while (key) {
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "END");
zyre_whisper (self->node, key, &msg);
key = (char *) zlist_next (keys);
}
zlist_destroy (&keys);
// wait for shout to be delivered
zclock_sleep (100);
ztask_packetizer_summary(self->packetizer, 0);
zyre_stop (self->node);
}
