static void
zyre_node_dump (zyre_node_t *self)
{
zsys_info ("zyre_node: dump state");
zsys_info (" - name=%s uuid=%s", self->name, zuuid_str (self->uuid));
zsys_info (" - endpoint=%s", self->endpoint);
if (self->beacon_port)
zsys_info (" - discovery=beacon port=%d interval=%zu",
self->beacon_port, self->interval);
else {
zsys_info (" - discovery=gossip");
if (self->gossip_bind)
zsys_info (" - bind endpoint=%s", self->gossip_bind);
if (self->gossip_connect)
zsys_info (" - connect endpoint=%s", self->gossip_connect);
}
zsys_info (" - headers=%zu:", zhash_size (self->headers));
zhash_foreach (self->headers, (zhash_foreach_fn *) zyre_node_log_pair, self);
zsys_info (" - peers=%zu:", zhash_size (self->peers));
zhash_foreach (self->peers, (zhash_foreach_fn *) zyre_node_log_item, self);
zsys_info (" - groups=%zu:", zhash_size (self->own_groups));
zhash_foreach (self->own_groups, (zhash_foreach_fn *) zyre_node_log_item, self);
}
static void
server_agent (void *args, zctx_t *ctx, void *pipe)
{
agent_t *self = agent_new (ctx, pipe);
zmq_pollitem_t items [] = {
{ self->pipe, 0, ZMQ_POLLIN, 0 },
{ self->router, 0, ZMQ_POLLIN, 0 }
};
while (!self->stopped) {
// Calculate tickless timer, up to 1 hour
uint64_t tickless = zclock_time () + 1000 * 3600;
zhash_foreach (self->clients, client_tickless, &tickless);
// Poll until at most next timer event
int rc = zmq_poll (items, 2,
(tickless - zclock_time ()) * ZMQ_POLL_MSEC);
if (rc == -1)
break; // Context has been shut down
// Process incoming message from either socket
if (items [0].revents & ZMQ_POLLIN)
agent_control_message (self);
if (items [1].revents & ZMQ_POLLIN)
agent_client_message (self);
// Send heartbeats to idle clients as needed
zhash_foreach (self->clients, client_ping, self->router);
}
agent_destroy (&self);
}
static zre_peer_t *
s_require_peer (agent_t *self, char *identity, char *address, uint16_t port)
{
zre_peer_t *peer = (zre_peer_t *) zhash_lookup (self->peers, identity);
if (!peer) {
// Purge any previous peer on same endpoint
char endpoint [100];
snprintf (endpoint, 100, "%s:%hu", address, port);
zhash_foreach (self->peers, agent_peer_purge, endpoint);
peer = zre_peer_new (identity, self->peers, self->ctx);
zre_peer_connect (peer, self->identity, endpoint);
// Handshake discovery by sending HELLO as first message
zre_msg_t *msg = zre_msg_new (ZRE_MSG_HELLO);
zre_msg_ipaddress_set (msg, zre_udp_host (self->udp));
zre_msg_mailbox_set (msg, self->port);
zre_msg_groups_set (msg, zhash_keys (self->own_groups));
zre_msg_status_set (msg, self->status);
zre_msg_headers_set (msg, zhash_dup (self->headers));
zre_peer_send (peer, &msg);
zre_log_info (self->log, ZRE_LOG_MSG_EVENT_ENTER,
zre_peer_endpoint (peer), endpoint);
// Now tell the caller about the peer
zstr_sendm (self->pipe, "ENTER");
zstr_send (self->pipe, identity);
}
return peer;
}
static zyre_peer_t *
zyre_node_require_peer (zyre_node_t *self, zuuid_t *uuid, const char *endpoint)
{
assert (self);
assert (endpoint);
zyre_peer_t *peer = (zyre_peer_t *) zhash_lookup (self->peers, zuuid_str (uuid));
if (!peer) {
// Purge any previous peer on same endpoint
zhash_foreach (self->peers, (zhash_foreach_fn *) zyre_node_purge_peer, (char *) endpoint);
peer = zyre_peer_new (self->peers, uuid);
assert (peer);
zyre_peer_set_origin (peer, self->name);
zyre_peer_set_verbose (peer, self->verbose);
zyre_peer_connect (peer, self->uuid, endpoint);
// Handshake discovery by sending HELLO as first message
zlist_t *groups = zhash_keys (self->own_groups);
zhash_t *headers = zhash_dup (self->headers);
zre_msg_t *msg = zre_msg_new (ZRE_MSG_HELLO);
zre_msg_set_endpoint (msg, self->endpoint);
zre_msg_set_groups (msg, &groups);
zre_msg_set_status (msg, self->status);
zre_msg_set_name (msg, self->name);
zre_msg_set_headers (msg, &headers);
zyre_peer_send (peer, &msg);
}
return peer;
}
static int
agent_ping_peer (const char *key, void *item, void *argument)
{
agent_t *self = (agent_t *) argument;
zre_peer_t *peer = (zre_peer_t *) item;
char *identity = zre_peer_identity (peer);
if (zclock_time () >= zre_peer_expired_at (peer)) {
zre_log_info (self->log, ZRE_LOG_MSG_EVENT_EXIT,
zre_peer_endpoint (peer),
zre_peer_endpoint (peer));
// If peer has really vanished, expire it
zstr_sendm (self->pipe, "EXIT");
zstr_send (self->pipe, identity);
zhash_foreach (self->peer_groups, agent_peer_delete, peer);
zhash_delete (self->peers, identity);
}
else
if (zclock_time () >= zre_peer_evasive_at (peer)) {
// If peer is being evasive, force a TCP ping.
// TODO: do this only once for a peer in this state;
// it would be nicer to use a proper state machine
// for peer management.
zre_msg_t *msg = zre_msg_new (ZRE_MSG_PING);
zre_peer_send (peer, &msg);
}
return 0;
}
static int exit_event_cb (flux_t h, int typemask, zmsg_t **zmsg, void *arg)
{
ctx_t *ctx = arg;
barrier_t *b;
json_object *o = NULL;
const char *name;
int errnum;
if (flux_msg_decode (*zmsg, NULL, &o) < 0 || o == NULL
|| util_json_object_get_string (o, "name", &name) < 0
|| util_json_object_get_int (o, "errnum", &errnum) < 0) {
flux_log (h, LOG_ERR, "%s: bad message", __FUNCTION__);
goto done;
}
if ((b = zhash_lookup (ctx->barriers, name))) {
b->errnum = errnum;
zhash_foreach (b->clients, send_enter_response, b);
zhash_delete (ctx->barriers, name);
}
done:
if (o)
json_object_put (o);
if (*zmsg)
zmsg_destroy (zmsg);
return 0;
}
static int
s_flush_ttl (zloop_t *loop, zmq_pollitem_t *poller, void *args)
{
clonesrv_t *self = (clonesrv_t *) args;
zhash_foreach (self->kvmap, s_flush_single, args);
return 0;
}
static void exit_event_cb (flux_t *h, flux_msg_handler_t *w,
const flux_msg_t *msg, void *arg)
{
ctx_t *ctx = arg;
barrier_t *b;
const char *json_str;
json_object *o = NULL;
const char *name;
int errnum;
if (flux_event_decode (msg, NULL, &json_str) < 0) {
flux_log_error (h, "%s: decoding event", __FUNCTION__);
goto done;
}
if (!(o = Jfromstr (json_str))
|| !Jget_str (o, "name", &name)
|| !Jget_int (o, "errnum", &errnum)) {
errno = EPROTO;
flux_log_error (h, "%s: decoding event", __FUNCTION__);
goto done;
}
if ((b = zhash_lookup (ctx->barriers, name))) {
b->errnum = errnum;
zhash_foreach (b->clients, send_enter_response, b);
zhash_delete (ctx->barriers, name);
}
done:
Jput (o);
}
static int
s_snapshots (zloop_t *loop, zmq_pollitem_t *poller, void *args)
{
clonesrv_t *self = (clonesrv_t *) args;
zframe_t *identity = zframe_recv (poller->socket);
if (identity) {
// Request is in second frame of message
char *request = zstr_recv (poller->socket);
char *subtree = NULL;
if (streq (request, "ICANHAZ?")) {
free (request);
subtree = zstr_recv (poller->socket);
}
else
printf ("E: bad request, aborting\n");
if (subtree) {
// Send state socket to client
kvroute_t routing = { poller->socket, identity, subtree };
zhash_foreach (self->kvmap, s_send_single, &routing);
// Now send END message with sequence number
zclock_log ("I: sending shapshot=%d", (int) self->sequence);
zframe_send (&identity, poller->socket, ZFRAME_MORE);
kvmsg_t *kvmsg = kvmsg_new (self->sequence);
kvmsg_set_key (kvmsg, "KTHXBAI");
kvmsg_set_body (kvmsg, (byte *) subtree, 0);
kvmsg_send (kvmsg, poller->socket);
kvmsg_destroy (&kvmsg);
free (subtree);
}
}
return 0;
}
static zyre_peer_t *
zyre_node_require_peer (zyre_node_t *self, zuuid_t *uuid, char *address, uint16_t port)
{
zyre_peer_t *peer = (zyre_peer_t *) zhash_lookup (self->peers, zuuid_str (uuid));
if (!peer) {
// Purge any previous peer on same endpoint
char endpoint [100];
snprintf (endpoint, 100, "tcp://%s:%hu", address, port);
zhash_foreach (self->peers, zyre_node_purge_peer, endpoint);
peer = zyre_peer_new (self->ctx, self->peers, uuid);
zyre_peer_connect (peer, self->uuid, endpoint);
// Handshake discovery by sending HELLO as first message
zre_msg_t *msg = zre_msg_new (ZRE_MSG_HELLO);
zre_msg_set_ipaddress (msg, self->host);
zre_msg_set_mailbox (msg, self->port);
zre_msg_set_groups (msg, zhash_keys (self->own_groups));
zre_msg_set_status (msg, self->status);
zre_msg_set_headers (msg, zhash_dup (self->headers));
zyre_peer_send (peer, &msg);
// Send new peer event to logger, if any
zyre_log_info (self->log, ZRE_LOG_MSG_EVENT_ENTER,
zyre_peer_endpoint (peer), endpoint);
}
return peer;
}
void
zyre_group_send (zyre_group_t *self, zre_msg_t **msg_p)
{
assert (self);
zhash_foreach (self->peers, (zhash_foreach_fn *) s_peer_send, *msg_p);
zre_msg_destroy (msg_p);
}
static void
state_manager (void *args, zctx_t *ctx, void *pipe)
{
zhash_t *kvmap = zhash_new ();
zstr_send (pipe, "READY");
void *snapshot = zsocket_new (ctx, ZMQ_ROUTER);
zsocket_bind (snapshot, "tcp://*:5556");
zmq_pollitem_t items [] = {
{ pipe, 0, ZMQ_POLLIN, 0 },
{ snapshot, 0, ZMQ_POLLIN, 0 }
};
int64_t sequence = 0; // Current snapshot version number
while (!zctx_interrupted) {
int rc = zmq_poll (items, 2, -1);
if (rc == -1 && errno == ETERM)
break; // Context has been shut down
// Apply state update from main thread
if (items [0].revents & ZMQ_POLLIN) {
kvmsg_t *kvmsg = kvmsg_recv (pipe);
if (!kvmsg)
break; // Interrupted
sequence = kvmsg_sequence (kvmsg);
kvmsg_store (&kvmsg, kvmap);
}
// Execute state snapshot request
if (items [1].revents & ZMQ_POLLIN) {
zframe_t *identity = zframe_recv (snapshot);
if (!identity)
break; // Interrupted
// Request is in second frame of message
char *request = zstr_recv (snapshot);
if (streq (request, "ICANHAZ?"))
free (request);
else {
printf ("E: bad request, aborting\n");
break;
}
// Send state snapshot to client
kvroute_t routing = { snapshot, identity };
// For each entry in kvmap, send kvmsg to client
zhash_foreach (kvmap, s_send_single, &routing);
// Now send END message with sequence number
printf ("Sending state shapshot=%d\n", (int) sequence);
zframe_send (&identity, snapshot, ZFRAME_MORE);
kvmsg_t *kvmsg = kvmsg_new (sequence);
kvmsg_set_key (kvmsg, "KTHXBAI");
kvmsg_set_body (kvmsg, (byte *) "", 0);
kvmsg_send (kvmsg, snapshot);
kvmsg_destroy (&kvmsg);
}
}
zhash_destroy (&kvmap);
}
static int
s_flush_ttl (zloop_t *loop, int timer_id, void *args)
{
clonesrv_t *self = (clonesrv_t *) args;
if (self->kvmap)
zhash_foreach (self->kvmap, s_flush_single, args);
return 0;
}
void
zcert_fprint (zcert_t *self, FILE *file)
{
assert (self);
fprintf (file, "metadata\n");
zhash_foreach (self->metadata, s_print_metadata, file);
fprintf (file, "curve\n");
fprintf (file, " public-key = \"%s\"\n", self->public_txt);
fprintf (file, " secret-key = \"%s\"\n", self->secret_txt);
}
static void timeout_cb (flux_reactor_t *r, flux_watcher_t *w,
int revents, void *arg)
{
ctx_t *ctx = arg;
assert (ctx->rank != 0);
ctx->timer_armed = false; /* one shot */
zhash_foreach (ctx->barriers, timeout_reduction, ctx);
}
static sub_t *
sub_new (client_t *client, char *path, zhash_t *cache)
{
sub_t *self = (sub_t *) zmalloc (sizeof (sub_t));
self->client = client;
self->path = strdup (path);
self->cache = zhash_dup (cache);
zhash_foreach (self->cache, s_resolve_cache_path, self);
return self;
}
static int timeout_cb (flux_t h, void *arg)
{
ctx_t *ctx = arg;
assert (!flux_treeroot (h));
ctx->timer_armed = false; /* one shot */
zhash_foreach (ctx->barriers, timeout_reduction, ctx);
return 0;
}
static void disconnect_request_cb (flux_t *h, flux_msg_handler_t *w,
const flux_msg_t *msg, void *arg)
{
ctx_t *ctx = arg;
char *sender;
if (flux_msg_get_route_first (msg, &sender) < 0)
return;
zhash_foreach (ctx->barriers, disconnect, sender);
free (sender);
}
static void
s_save_metadata_all (zcert_t *self)
{
zconfig_destroy (&self->config);
self->config = zconfig_new ("root", NULL);
zconfig_t *section = zconfig_new ("metadata", self->config);
zhash_foreach (self->metadata, s_save_metadata, section);
char *timestr = zclock_timestr ();
zconfig_comment (self->config, " **** Generated on %s by CZMQ ****", timestr);
zstr_free (×tr);
}
static int disconnect_request_cb (flux_t h, int typemask, zmsg_t **zmsg,
void *arg)
{
ctx_t *ctx = arg;
char *sender = flux_msg_sender (*zmsg);
if (sender) {
zhash_foreach (ctx->barriers, disconnect, sender);
free (sender);
}
zmsg_destroy (zmsg);
return 0;
}
static void
zyre_node_dump (zyre_node_t *self)
{
fflush (stdout);
printf ("************** zyre_node_dump *************************\n");
printf ("node id : %s\n", zuuid_str (self->uuid));
printf (" endpoint = %s\n", self->endpoint);
printf (" headers [%zu] { \n", zhash_size (self->headers));
zhash_foreach (self->headers, zyre_node_hash_key_dump, self);
printf (" }\n");
printf (" peers [%zu] {\n", zhash_size (self->peers));
zhash_foreach (self->peers, zyre_node_hash_key_dump, self);
printf (" }\n");
printf (" own groups [%zu] { \n", zhash_size (self->own_groups));
zhash_foreach (self->own_groups, zyre_node_hash_key_dump, self);
printf (" }\n");
printf (" peer groups [%zu] {\n", zhash_size (self->peer_groups));
zhash_foreach (self->peer_groups, zyre_node_hash_key_dump, self);
printf (" }\n");
printf ("*******************************************************\n");
fflush (stdout);
}
static void
monitor_the_server (server_t *self, client_t *client)
{
bool activity = false;
mount_t *mount = (mount_t *) zlist_first (self->mounts);
while (mount) {
if (mount_refresh (mount, self))
activity = true;
mount = (mount_t *) zlist_next (self->mounts);
}
if (activity)
zhash_foreach (self->clients, client_dispatch, self);
}
void
zyre_event_print (zyre_event_t *self)
{
zsys_info ("zyre_event:");
zsys_info (" - from name=%s uuid=%s", zyre_event_name(self), zyre_event_sender(self));
switch (self->type) {
case ZYRE_EVENT_ENTER:
zsys_info (" - type=ENTER");
zsys_info (" - headers=%zu:", zhash_size (self->headers));
zhash_foreach (self->headers, (zhash_foreach_fn *) zyre_event_log_pair, self);
zsys_info (" - address=%s", zyre_event_address(self));
break;
case ZYRE_EVENT_EXIT:
zsys_info (" - type=EXIT");
break;
case ZYRE_EVENT_STOP:
zsys_info (" - type=STOP");
break;
case ZYRE_EVENT_JOIN:
zsys_info (" - type=JOIN");
zsys_info (" - group=%s", zyre_event_group(self));
break;
case ZYRE_EVENT_LEAVE:
zsys_info (" - type=LEAVE");
zsys_info (" - group=%s", zyre_event_group(self));
break;
case ZYRE_EVENT_SHOUT:
zsys_info (" - type=SHOUT");
zsys_info (" - message:");
zmsg_print (self->msg);
break;
case ZYRE_EVENT_WHISPER:
zsys_info (" - type=WHISPER");
zsys_info (" - message:");
zmsg_print (self->msg);
break;
case ZYRE_EVENT_EVASIVE:
zsys_info (" - type=EVASIVE");
break;
default:
zsys_info (" - type=UNKNOWN");
break;
}
}
// Finally here's the server thread itself, which polls its two
// sockets and processes incoming messages
static void
server_thread (void *args, zctx_t *ctx, void *pipe)
{
server_t *self = server_new (ctx, pipe);
zmq_pollitem_t items [] = {
{ self->pipe, 0, ZMQ_POLLIN, 0 },
{ self->router, 0, ZMQ_POLLIN, 0 }
};
self->monitor_at = zclock_time () + self->monitor;
while (!self->stopped && !zctx_interrupted) {
// Calculate tickless timer, up to interval seconds
uint64_t tickless = zclock_time () + self->monitor;
zhash_foreach (self->clients, client_tickless, &tickless);
// Poll until at most next timer event
int rc = zmq_poll (items, 2,
(tickless - zclock_time ()) * ZMQ_POLL_MSEC);
if (rc == -1)
break; // Context has been shut down
// Process incoming message from either socket
if (items [0].revents & ZMQ_POLLIN)
server_control_message (self);
if (items [1].revents & ZMQ_POLLIN)
server_client_message (self);
// Send heartbeats to idle clients as needed
zhash_foreach (self->clients, client_ping, self);
// If clock went past timeout, then monitor server
if (zclock_time () >= self->monitor_at) {
monitor_the_server (self, NULL);
self->monitor_at = zclock_time () + self->monitor;
}
}
server_destroy (&self);
}
static int disconnect_request_cb (flux_t h, int typemask, zmsg_t **zmsg,
void *arg)
{
ctx_t *ctx = arg;
char *sender;
if (flux_msg_get_route_first (*zmsg, &sender) < 0)
goto done;
zhash_foreach (ctx->barriers, disconnect, sender);
free (sender);
done:
zmsg_destroy (zmsg);
return 0;
}
void
zyre_node_actor (zsock_t *pipe, void *args)
{
// Create node instance to pass around
zyre_node_t *self = zyre_node_new (pipe, args);
if (!self) // Interrupted
return;
// Signal actor successfully initialized
zsock_signal (self->pipe, 0);
// Loop until the agent is terminated one way or another
int64_t reap_at = zclock_time () + REAP_INTERVAL;
while (!self->terminated) {
int timeout = (int) (reap_at - zclock_time ());
if (timeout > REAP_INTERVAL)
timeout = REAP_INTERVAL;
else
if (timeout < 0)
timeout = 0;
zsock_t *which = (zsock_t *) zpoller_wait (self->poller, timeout);
if (which == self->pipe)
zyre_node_recv_api (self);
else
if (which == self->inbox)
zyre_node_recv_peer (self);
else
if (self->beacon
&& (void *) which == self->beacon)
zyre_node_recv_beacon (self);
else
if (self->gossip
&& (zactor_t *) which == self->gossip)
zyre_node_recv_gossip (self);
else
if (zpoller_terminated (self->poller))
break; // Interrupted, check before expired
else
if (zpoller_expired (self->poller)) {
if (zclock_time () >= reap_at) {
reap_at = zclock_time () + REAP_INTERVAL;
// Ping all peers and reap any expired ones
zhash_foreach (self->peers, (zhash_foreach_fn *) zyre_node_ping_peer, self);
}
}
}
zyre_node_destroy (&self);
}
static void
zyre_node_remove_peer (zyre_node_t *self, zyre_peer_t *peer)
{
// Tell the calling application the peer has gone
zstr_sendm (self->pipe, "EXIT");
zstr_send (self->pipe, zyre_peer_identity (peer));
// Send a log event
zyre_log_info (self->log, ZRE_LOG_MSG_EVENT_EXIT,
zyre_peer_endpoint (peer),
zyre_peer_endpoint (peer));
// Remove peer from any groups we've got it in
zhash_foreach (self->peer_groups, zyre_node_delete_peer, peer);
// To destroy peer, we remove from peers hash table
zhash_delete (self->peers, zyre_peer_identity (peer));
}
void
zyre_node_engine (void *args, zctx_t *ctx, void *pipe)
{
// Create node instance to pass around
zyre_node_t *self = zyre_node_new (ctx, pipe);
if (!self) // Interrupted
return;
zsocket_signal (self->pipe);
uint64_t reap_at = zclock_time () + REAP_INTERVAL;
zpoller_t *poller = zpoller_new (
self->pipe, self->inbox, zbeacon_socket (self->beacon), NULL);
while (!zpoller_terminated (poller)) {
if (self->terminated)
break;
int timeout = (int) (reap_at - zclock_time ());
assert (timeout <= REAP_INTERVAL);
if (timeout < 0)
timeout = 0;
void *which = zpoller_wait (poller, timeout);
if (which == self->pipe)
zyre_node_recv_api (self);
else
if (which == self->inbox)
zyre_node_recv_peer (self);
else
if (which == zbeacon_socket (self->beacon))
zyre_node_recv_beacon (self);
else
if (zpoller_expired(poller)) {
if (zclock_time () >= reap_at) {
reap_at = zclock_time () + REAP_INTERVAL;
// Ping all peers and reap any expired ones
zhash_foreach (self->peers, zyre_node_ping_peer, self);
}
}
else
if (zpoller_terminated(poller))
break;
else
// This should never happen
assert(false);
}
zpoller_destroy (&poller);
zyre_node_destroy (&self);
}
static void
zre_node_agent (void *args, zctx_t *ctx, void *pipe)
{
// Create agent instance to pass around
agent_t *self = agent_new (ctx, pipe);
if (!self) // Interrupted
return;
// Send first beacon immediately
uint64_t ping_at = zclock_time ();
zmq_pollitem_t pollitems [] = {
{ self->pipe, 0, ZMQ_POLLIN, 0 },
{ self->inbox, 0, ZMQ_POLLIN, 0 },
{ 0, zre_udp_handle (self->udp), ZMQ_POLLIN, 0 },
{ fmq_client_handle (self->fmq_client), 0, ZMQ_POLLIN, 0 }
};
while (!zctx_interrupted) {
long timeout = (long) (ping_at - zclock_time ());
assert (timeout <= PING_INTERVAL);
if (timeout < 0)
timeout = 0;
if (zmq_poll (pollitems, 4, timeout * ZMQ_POLL_MSEC) == -1)
break; // Interrupted
if (pollitems [0].revents & ZMQ_POLLIN)
agent_recv_from_api (self);
if (pollitems [1].revents & ZMQ_POLLIN)
agent_recv_from_peer (self);
if (pollitems [2].revents & ZMQ_POLLIN)
agent_recv_udp_beacon (self);
if (pollitems [3].revents & ZMQ_POLLIN)
agent_recv_fmq_event (self);
if (zclock_time () >= ping_at) {
agent_beacon_send (self);
ping_at = zclock_time () + PING_INTERVAL;
// Ping all peers and reap any expired ones
zhash_foreach (self->peers, agent_ping_peer, self);
}
}
agent_destroy (&self);
}
static void
zyre_node_remove_peer (zyre_node_t *self, zyre_peer_t *peer)
{
// Tell the calling application the peer has gone
zstr_sendm (self->outbox, "EXIT");
zstr_sendm (self->outbox, zyre_peer_identity (peer));
zstr_send (self->outbox, zyre_peer_name (peer));
if (self->verbose)
zsys_info ("(%s) EXIT name=%s endpoint=%s",
self->name, zyre_peer_name (peer), zyre_peer_endpoint (peer));
// Remove peer from any groups we've got it in
zhash_foreach (self->peer_groups, (zhash_foreach_fn *) zyre_node_delete_peer, peer);
// To destroy peer, we remove from peers hash table
zhash_delete (self->peers, zyre_peer_identity (peer));
}