void
ztask_job_request_clean (ztask_job_request_t *self, ztask_node_manager_t *node_mgr)
{
assert (self);
// zclock_log ("Cleaning job request from %s ...", zyre_event_sender (self->request));
zlist_t *keys = zhash_keys (self->processes);
char *key = (char *) zlist_first (keys);
ztask_job_proc_t *p;
int pid;
while (key) {
p = (ztask_job_proc_t *) zhash_lookup (self->processes, key);
zhash_delete (self->processes, key);
pid = ztask_job_proc_pid(p);
// assert (pid);
if (pid) {
zclock_log("Killing pid=%d ...", ztask_job_proc_pid(p));
kill (ztask_job_proc_pid(p), SIGKILL);
zmsg_t *msg_report = zmsg_new ();
zmsg_addstr (msg_report, "REPORT");
zmsg_addstr (msg_report, ztask_job_proc_jobid(p));
zmsg_addstr (msg_report, "-100");
zyre_whisper (ztask_node_manager_zyre_node(node_mgr), zyre_event_sender(self->request), &msg_report);
zmsg_destroy (&msg_report);
zhash_delete (ztask_node_manager_list_running_processes (node_mgr), ztask_job_proc_jobid(p));
zlist_append (ztask_node_manager_list_available_processes (node_mgr), p);
ztask_job_proc_reset(p);
}
key = (char *) zlist_next (keys);
}
zlist_destroy (&keys);
}
static void
pipe_drop_local_writer (pipe_t **self_p)
{
assert (self_p);
if (*self_p) {
pipe_t *self = *self_p;
// TODO: what if self->writer is REMOTE_NODE?
self->writer = NULL;
if (self->reader) {
if (self->reader == REMOTE_NODE) {
// Tell remote node we're dropping off
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "DROP WRITER");
zmsg_addstr (msg, self->name);
zyre_whisper (self->server->zyre, self->remote, &msg);
zsys_info ("%s: tell peer we stopped being writer", self->name);
}
else {
engine_send_event (self->reader, writer_dropped_event);
// Don't destroy pipe yet - reader is still using it
*self_p = NULL;
}
}
pipe_destroy (self_p);
}
}
static int
pipe_attach_remote_writer (pipe_t *self, const char *remote, bool unicast)
{
assert (self);
if (self->reader == REMOTE_NODE) {
// We're witnessing two nodes chatting, so we can drop the pipe
// and forget all about it
pipe_destroy (&self);
return 0;
}
else
if (self->writer == NULL) {
// This is how we indicate a remote writer
self->writer = REMOTE_NODE;
self->remote = strdup (remote);
zsys_info ("%s: attach remote writer", self->name);
if (self->reader && !unicast) {
// Tell remote node we're acting as reader, if we got a
// broadcast message. If we got a unicast message, the peer
// already knows about us, so don't re-echo the message
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "HAVE READER");
zmsg_addstr (msg, self->name);
zyre_whisper (self->server->zyre, self->remote, &msg);
zsys_info ("%s: tell peer we are now reader", self->name);
}
return 0;
}
zsys_info ("%s: pipe already has writer: ignored", self->name);
return -1;
}
JNIEXPORT jint JNICALL
Java_org_zeromq_zyre_Zyre__1_1whisper (JNIEnv *env, jclass c, jlong self, jstring peer, jlong msg_p)
{
char *peer_ = (char *) (*env)->GetStringUTFChars (env, peer, NULL);
jint whisper_ = (jint) zyre_whisper ((zyre_t *) (intptr_t) self, peer_, (zmsg_t **) (intptr_t) &msg_p);
(*env)->ReleaseStringUTFChars (env, peer, peer_);
return whisper_;
}
void
zsync_node_recv_from_agent (zsync_node_t *self)
{
assert (self);
zsync_msg_t *msg = zsync_msg_recv (self->zsync_pipe);
switch (zsync_msg_id (msg)) {
case ZSYNC_MSG_REQ_FILES: {
char *receiver = zsync_msg_receiver (msg);
char *zyre_uuid = zsync_node_zyre_uuid (self, receiver);
if (zyre_uuid) {
uint64_t size = zsync_msg_size (msg);
printf("[ND] Recv Agent WHISPER REQUEST %s ; %s\n", zyre_uuid, receiver);
zmsg_t *zyre_out = zmsg_new ();
zs_msg_pack_request_files (zyre_out, zsync_msg_files (msg));
zyre_whisper (self->zyre, zyre_uuid, &zyre_out);
zsync_credit_msg_send_request (self->credit_pipe, zyre_uuid, size);
}
break;
}
case ZSYNC_MSG_UPDATE:
printf("[ND] Recv Agent SHOUT UPDATE\n");
zmsg_t *zyre_out = zsync_msg_update_msg (msg);
zyre_shout (self->zyre, "ZSYNC", &zyre_out);
break;
case ZSYNC_MSG_TERMINATE:
zyre_stop (self->zyre);
// terminate file transfer manager
zsync_ftm_msg_send_terminate (self->file_pipe);
// terminate credit manager
zsync_credit_msg_send_terminate (self->credit_pipe);
// receive termination confirmation
msg = zsync_msg_recv (self->file_pipe);
zsync_msg_destroy (&msg);
printf("OK ft\n");
msg = zsync_msg_recv (self->credit_pipe);
zsync_msg_destroy (&msg);
printf("OK cm\n");
// send shutdown confirmation to agent
zsync_msg_send_terminate (self->zsync_pipe);
self->terminated = true;
break;
}
}
static void
pipe_send_data (pipe_t *self, zchunk_t **chunk_p)
{
assert (self);
assert (self->reader);
zchunk_t *chunk = *chunk_p;
assert (chunk);
if (self->reader == REMOTE_NODE) {
// Send chunk to remote node reader
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "DATA");
zmsg_addstr (msg, self->name);
zmsg_addmem (msg, zchunk_data (chunk), zchunk_size (chunk));
zyre_whisper (self->server->zyre, self->remote, &msg);
zchunk_destroy (chunk_p);
}
else {
client_store_chunk (self->reader, chunk_p);
engine_send_event (self->reader, have_data_event);
}
}
static int
pipe_attach_local_writer (pipe_t *self, client_t *writer)
{
assert (self);
if (self->writer == NULL) {
zsys_info ("%s: attach local writer", self->name);
self->writer = writer;
if (self->reader == NULL) {
if (self->server->zyre) {
// Announce that we have a new pipe writer so that readers
// in the cluster may discover us
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "HAVE WRITER");
zmsg_addstr (msg, self->name);
zyre_shout (self->server->zyre, "ZPIPES", &msg);
zsys_info ("%s: broadcast we are now writer", self->name);
}
}
else
if (self->reader == REMOTE_NODE) {
assert (self->server->zyre);
// Tell remote node we would like to be writer
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "HAVE WRITER");
zmsg_addstr (msg, self->name);
zyre_whisper (self->server->zyre, self->remote, &msg);
zsys_info ("%s: tell peer we are now writer", self->name);
}
else
engine_send_event (self->reader, have_writer_event);
return 0;
}
zsys_info ("%s: pipe already has writer: ignored", self->name);
return -1;
}
///
// Send message to single peer, specified as a UUID string
// Destroys message after sending
int QmlZyre::whisper (const QString &peer, zmsg_t **msgP) {
return zyre_whisper (self, peer.toUtf8().data(), msgP);
};
///
// Send message to single peer, specified as a UUID string
// Destroys message after sending
int QZyre::whisper (const QString &peer, QZmsg *msgP)
{
int rv = zyre_whisper (self, peer.toUtf8().data(), &msgP->self);
return rv;
}
///
// Send message to single peer, specified as a UUID string
// Destroys message after sending
int QZyre::whisper (const QString &peer, zmsg_t **msgP)
{
int rv = zyre_whisper (self, peer.toUtf8().data(), msgP);
return rv;
}
void whisper(const std::string& peer, zmsg_t* msg) const
{
zyre_whisper(m_self, peer.c_str(), &msg);
}
void
ztask_run_manager_loop (ztask_run_manager_t *self)
{
assert (self);
if (!self->packetizer) {
zclock_log ("E: No packetizer set !!!");
return;
}
// Setting network interface if neede
if (self->node_interface)
zyre_set_interface(self->node, self->node_interface);
zyre_set_header (self->node, "X-ZTASK-RUN", "ZTASK RUN");
zyre_start (self->node);
zyre_dump (self->node);
zclock_sleep (10);
// ztask_packetizer_dump (self->packetizer);
ztask_packet_t *packet;
int request_sent = 0;
zyre_event_t *event;
while (!zsys_interrupted) {
if (ztask_packetizer_is_finished (self->packetizer))
break;
event = zyre_event_new (self->node);
if (!event)
break;
if (zyre_event_type (event) == ZYRE_EVENT_ENTER) {
// Ignoring nodes which don't have service X-ZTASK-NODE defined
if (zyre_event_header (event, "X-ZTASK-NODE")) {
zhash_insert (self->nodes, zyre_event_sender (event),"");
ztask_log_debug (self->log, "Adding node -> workers=%ld", zhash_size (self->nodes));
} else {
// TODO disconnect worker (zyre peer) somehow
}
}
else
if (zyre_event_type (event) == ZYRE_EVENT_EXIT) {
if (zhash_lookup (self->nodes, zyre_event_sender (event))) {
ztask_log_debug (self->log, "Removing node -> workers=%ld", zhash_size (self->nodes));
zhash_delete (self->nodes, zyre_event_sender (event));
// Cleanup packetizer in case ztask_node was killed
request_sent -= ztask_packetizer_running_node_cleanup (self->packetizer, zyre_event_sender (event));
}
}
else
if (zyre_event_type (event) == ZYRE_EVENT_WHISPER) {
// Ingoring whispers when they are not from our nodes
if (!zhash_lookup(self->nodes, zyre_event_sender (event))) {
ztask_log_warning (self->log, "W: Ingoring whisper from %s", zyre_event_sender (event));
zyre_event_destroy (&event);
continue;
}
zmsg_t *msg_report = zyre_event_msg(event);
char *str_msg = zmsg_popstr (msg_report);
if (streq(str_msg, "REQUEST")) {
// Let's handle request
packet = ztask_packetizer_next_packet (self->packetizer);
if (packet) {
char *cmd;
if ( asprintf (&cmd, "%s", ztask_packet_cmd (packet)) < 0)
cmd = NULL;
assert (cmd);
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "REQUEST");
zmsg_addstrf (msg, "%s_%ld", zyre_uuid (self->node), ztask_packet_id (packet));
zmsg_addstr (msg, cmd);
zyre_whisper (self->node, zyre_event_sender (event), &msg);
ztask_packet_set_node (packet, zyre_event_sender (event));
request_sent++;
ztask_packetizer_info (self->packetizer, request_sent);
ztask_log_debug (self->log, "ID=%s_%ld cmd='%s' running=%d", zyre_uuid (self->node), ztask_packet_id (packet), cmd, request_sent);
free (cmd);
}
else {
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "NO_PACKETS");
zyre_whisper (self->node, zyre_event_sender (event), &msg);
}
}
else
if (streq(str_msg, "REPORT")) {
// It's report
char *str_id = zmsg_popstr (msg_report);
char *str_pid_rc = zmsg_popstr (msg_report);
ztask_log_debug (self->log, "REPORT ID=%s rc=%s", str_id, str_pid_rc);
ztask_packetizer_report (self->packetizer, str_id, str_pid_rc);
request_sent--;
ztask_packetizer_info (self->packetizer, request_sent);
free (str_id);
free (str_pid_rc);
}
else {
ztask_log_error (self->log, "E: ztask_run_manager_loop : Wrong message %s", str_msg);
assert (false);
}
free (str_msg);
}
zyre_event_destroy (&event);
}
// FIXME : simplify zhash_foreach()
zlist_t *keys = zhash_keys (self->nodes);
char *key = (char *) zlist_first (keys);
while (key) {
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "END");
zyre_whisper (self->node, key, &msg);
key = (char *) zlist_next (keys);
}
zlist_destroy (&keys);
// wait for shout to be delivered
zclock_sleep (100);
ztask_packetizer_summary(self->packetizer, 0);
zyre_stop (self->node);
}
void
zsync_node_engine (void *args, zctx_t *ctx, void *pipe)
{
int rc;
zsync_node_t *self = zsync_node_new ();
self->ctx = ctx;
self->zyre = zyre_new (ctx);
self->zsync_pipe = pipe;
// Join group
rc = zyre_join (self->zyre, "ZSYNC");
assert (rc == 0);
// Give time to interconnect
zclock_sleep (250);
zpoller_t *poller = zpoller_new (zyre_socket (self->zyre), self->zsync_pipe, NULL);
// Create thread for file management
self->file_pipe = zthread_fork (self->ctx, zsync_ftmanager_engine, NULL);
zpoller_add (poller, self->file_pipe);
// Create thread for credit management
self->credit_pipe = zthread_fork (self->ctx, zsync_credit_manager_engine, NULL);
zpoller_add (poller, self->credit_pipe);
// Start receiving messages
printf("[ND] started\n");
while (!zpoller_terminated (poller)) {
void *which = zpoller_wait (poller, -1);
if (which == zyre_socket (self->zyre)) {
zsync_node_recv_from_zyre (self);
}
else
if (which == self->zsync_pipe) {
printf("[ND] Recv Agent\n");
zsync_node_recv_from_agent (self);
}
else
if (which == self->file_pipe) {
printf("[ND] Recv FT Manager\n");
zsync_ftm_msg_t *msg = zsync_ftm_msg_recv (self->file_pipe);
char *receiver = zsync_ftm_msg_receiver (msg);
char *zyre_uuid = zsync_node_zyre_uuid (self, receiver);
if (zyre_uuid) {
char *path = zsync_ftm_msg_path (msg);
uint64_t sequence = zsync_ftm_msg_sequence (msg);
uint64_t chunk_size = zsync_ftm_msg_chunk_size (msg);
uint64_t offset = zsync_ftm_msg_offset (msg);
zsync_msg_send_req_chunk (pipe, path, chunk_size, offset);
zsync_msg_t *zsmsg = zsync_msg_recv (pipe);
zchunk_t *chunk = zsync_msg_chunk (zsmsg);
zframe_t *frame = zframe_new (zchunk_data (chunk), zchunk_size (chunk));
zmsg_t *zmsg = zmsg_new ();
zs_msg_pack_chunk (zmsg, sequence, path, offset, frame);
zyre_whisper (self->zyre, zyre_uuid, &zmsg);
zsync_ftm_msg_destroy (&msg);
zsync_msg_destroy (&zsmsg);
}
}
else
if (which == self->credit_pipe) {
printf("[ND] Recv Credit Manager\n");
zsync_credit_msg_t *cmsg = zsync_credit_msg_recv (self->credit_pipe);
char *receiver = zsync_credit_msg_receiver (cmsg);
char *zyre_uuid = zsync_node_zyre_uuid (self, receiver);
if (zyre_uuid) {
zmsg_t *credit_msg = zsync_credit_msg_credit (cmsg);
assert (rc == 0);
zyre_whisper (self->zyre, zyre_uuid, &credit_msg);
}
zsync_credit_msg_destroy (&cmsg);
}
if (self->terminated) {
break;
}
}
zpoller_destroy (&poller);
zsync_node_destroy (&self);
printf("[ND] stopped\n");
}
static void
zsync_node_recv_from_zyre (zsync_node_t *self)
{
zsync_peer_t *sender;
char *zyre_sender;
zuuid_t *sender_uuid;
zmsg_t *zyre_in, *zyre_out, *fm_msg;
zlist_t *fpaths, *fmetadata;
zyre_event_t *event = zyre_event_recv (self->zyre);
zyre_sender = zyre_event_sender (event); // get tmp uuid
switch (zyre_event_type (event)) {
case ZYRE_EVENT_ENTER:
printf("[ND] ZS_ENTER: %s\n", zyre_sender);
zhash_insert (self->zyre_peers, zyre_sender, NULL);
break;
case ZYRE_EVENT_JOIN:
printf ("[ND] ZS_JOIN: %s\n", zyre_sender);
// Obtain own current state
zsync_msg_send_req_state (self->zsync_pipe);
zsync_msg_t *msg_state = zsync_msg_recv (self->zsync_pipe);
assert (zsync_msg_id (msg_state) == ZSYNC_MSG_RES_STATE);
uint64_t state = zsync_msg_state (msg_state);
// Send GREET message
zyre_out = zmsg_new ();
zs_msg_pack_greet (zyre_out, zuuid_data (self->own_uuid), state);
zyre_whisper (self->zyre, zyre_sender, &zyre_out);
break;
case ZYRE_EVENT_LEAVE:
break;
case ZYRE_EVENT_EXIT:
/*
printf("[ND] ZS_EXIT %s left the house!\n", zyre_sender);
sender = zhash_lookup (self->zyre_peers, zyre_sender);
if (sender) {
// Reset Managers
zmsg_t *reset_msg = zmsg_new ();
zmsg_addstr (reset_msg, zsync_peer_uuid (sender));
zmsg_addstr (reset_msg, "ABORT");
zmsg_send (&reset_msg, self->file_pipe);
reset_msg = zmsg_new ();
zmsg_addstr (reset_msg, zsync_peer_uuid (sender));
zmsg_addstr (reset_msg, "ABORT");
zmsg_send (&reset_msg, self->credit_pipe);
// Remove Peer from active list
zhash_delete (self->zyre_peers, zyre_sender);
}*/
break;
case ZYRE_EVENT_WHISPER:
case ZYRE_EVENT_SHOUT:
printf ("[ND] ZS_WHISPER: %s\n", zyre_sender);
sender = zhash_lookup (self->zyre_peers, zyre_sender);
zyre_in = zyre_event_msg (event);
zs_msg_t *msg = zs_msg_unpack (zyre_in);
switch (zs_msg_get_cmd (msg)) {
case ZS_CMD_GREET:
// Get perm uuid
sender_uuid = zuuid_new ();
zuuid_set (sender_uuid, zs_msg_uuid (msg));
sender = zsync_node_peers_lookup (self, zuuid_str (sender_uuid));
if (!sender) {
sender = zsync_peer_new (zuuid_str (sender_uuid), 0x0);
zlist_append (self->peers, sender);
}
assert (sender);
zhash_update (self->zyre_peers, zyre_sender, sender);
zsync_peer_set_zyre_state (sender, ZYRE_EVENT_JOIN);
// Get current state for sender
uint64_t remote_current_state = zs_msg_get_state (msg);
printf ("[ND] current state: %"PRId64"\n", remote_current_state);
// Lookup last known state
uint64_t last_state_local = zsync_peer_state (sender);
printf ("[ND] last known state: %"PRId64"\n", zsync_peer_state (sender));
// Send LAST_STATE if differs
if (remote_current_state >= last_state_local) {
zmsg_t *lmsg = zmsg_new ();
zs_msg_pack_last_state (lmsg, last_state_local);
zyre_whisper (self->zyre, zyre_sender, &lmsg);
}
break;
case ZS_CMD_LAST_STATE:
assert (sender);
zyre_out = zmsg_new ();
// Gets updates from client
uint64_t last_state_remote = zs_msg_get_state (msg);
zsync_msg_send_req_update (self->zsync_pipe, last_state_remote);
zsync_msg_t *msg_upd = zsync_msg_recv (self->zsync_pipe);
assert (zsync_msg_id (msg_upd) == ZSYNC_MSG_UPDATE);
// Send UPDATE
zyre_out = zsync_msg_update_msg (msg_upd);
zyre_whisper (self->zyre, zyre_sender, &zyre_out);
break;
case ZS_CMD_UPDATE:
printf ("[ND] UPDATE\n");
assert (sender);
uint64_t state = zs_msg_get_state (msg);
zsync_peer_set_state (sender, state);
zsync_node_save_peers (self);
fmetadata = zs_msg_get_fmetadata (msg);
zmsg_t *zsync_msg = zmsg_new ();
zs_msg_pack_update (zsync_msg, zs_msg_get_state (msg), fmetadata);
zsync_msg_send_update (self->zsync_pipe, zsync_peer_uuid (sender), zsync_msg);
break;
case ZS_CMD_REQUEST_FILES:
printf ("[ND] REQUEST FILES\n");
fpaths = zs_msg_fpaths (msg);
zmsg_t *fm_msg = zmsg_new ();
zmsg_addstr (fm_msg, zsync_peer_uuid (sender));
zmsg_addstr (fm_msg, "REQUEST");
char *fpath = zs_msg_fpaths_first (msg);
while (fpath) {
zmsg_addstr (fm_msg, fpath);
printf("[ND] %s\n", fpath);
fpath = zs_msg_fpaths_next (msg);
}
zmsg_send (&fm_msg, self->file_pipe);
break;
case ZS_CMD_GIVE_CREDIT:
printf("[ND] GIVE CREDIT\n");
fm_msg = zmsg_new ();
zmsg_addstr (fm_msg, zsync_peer_uuid (sender));
zmsg_addstr (fm_msg, "CREDIT");
zmsg_addstrf (fm_msg, "%"PRId64, zs_msg_get_credit (msg));
zmsg_send (&fm_msg, self->file_pipe);
break;
case ZS_CMD_SEND_CHUNK:
printf("[ND] SEND_CHUNK (RCV)\n");
// Send receival to credit manager
zframe_t *zframe = zs_msg_get_chunk (msg);
uint64_t chunk_size = zframe_size (zframe);
zsync_credit_msg_send_update (self->credit_pipe, zsync_peer_uuid (sender), chunk_size);
// Pass chunk to client
byte *data = zframe_data (zframe);
zchunk_t *chunk = zchunk_new (data, chunk_size);
char *path = zs_msg_get_file_path (msg);
uint64_t seq = zs_msg_get_sequence (msg);
uint64_t off = zs_msg_get_offset (msg);
zsync_msg_send_chunk (self->zsync_pipe, chunk, path, seq, off);
break;
case ZS_CMD_ABORT:
// TODO abort protocol managed file transfer
printf("[ND] ABORT\n");
break;
default:
assert (false);
break;
}
zs_msg_destroy (&msg);
break;
default:
printf("[ND] Error command not found\n");
break;
}
zyre_event_destroy (&event);
}
