int rrwrk_start(rrwrk_t *self, wrk_task_fn *cb)
{
self->heartbeat_at = zclock_time() + self->heartbeat;
self->cb = cb;
//** Start task thread and wait for synchronization signal
self->pipe = zthread_fork(self->ctx, rrtask_manager_fn, (void *)self);
assert(self->pipe);
free(zstr_recv(self->pipe));
//self->liveness = HEARTBEAT_LIVENESS; //** Don't do reconnect before the first connection established
while(!zctx_interrupted) {
zmq_pollitem_t items[] = {{self->worker, 0, ZMQ_POLLIN, 0}, {self->pipe, 0, ZMQ_POLLIN, 0}}; //** Be aware: this must be within while loop!!
int rc = zmq_poll(items, 2, self->heartbeat * ZMQ_POLL_MSEC);
if (rc == -1)
break; //** Interrupted
if (items[0].revents & ZMQ_POLLIN) { //** Data from broker is ready
zmsg_t *msg = zmsg_recv(self->worker);
if (!msg)
break; //** Interrupted. Need to do more research to confirm it
self->liveness = HEARTBEAT_LIVENESS;
self->last_heartbeat = zclock_time();
//** Dont try to handle errors, just assert noisily
assert(zmsg_size(msg) >= 3); //** empty + header + command + ...
zframe_t *empty = zmsg_pop(msg);
assert(zframe_streq(empty, ""));
zframe_destroy(&empty);
zframe_t *header = zmsg_pop(msg);
assert(zframe_streq(header, RR_WORKER));
zframe_destroy(&header);
zframe_t *command = zmsg_pop(msg);
if (zframe_streq(command, RRWRK_REQUEST)) {
assert(zmsg_size(msg) == 3); //** UUID + SOURCE + INPUT DATA
self->total_received++;
zmq_pollitem_t item = {self->pipe, 0, ZMQ_POLLOUT, 0};
int rc = zmq_poll(&item, 1, 0);
assert(rc != -1);
if (item.revents & ZMQ_POLLOUT) { //** Dispatch it if worker is ready
//** Send task to task manager
zmsg_send(&msg, self->pipe);
} else { //** Otherwise put it on waiting list
zlist_push(self->data, zmsg_dup(msg));
}
} else if (zframe_streq(command, RRWRK_HEARTBEAT)) {
; //** Do nothing for heartbeat
} else if (zframe_streq(command, RRWRK_DISCONNECT)) {
rrwrk_connect_to_broker(self);
} else {
log_printf(0, "E: invalid input message\n");
}
zframe_destroy(&command);
zmsg_destroy(&msg);
} else if ((zclock_time() - self->heartbeat) > self->last_heartbeat) {
if(--self->liveness == 0) {
rrwrk_print(self);
log_printf(0, "W: Disconnected from broker - retrying ...\n");
rrwrk_print(self);
zclock_sleep(self->reconnect);
rrwrk_connect_to_broker(self);
}
}
if (items[1].revents & ZMQ_POLLIN) { //** Data from pipe is ready
zmsg_t *output = zmsg_recv(self->pipe);
assert(zmsg_size(output) == 3); //** UUID + SOURCE + DATA
self->total_finished++;
zmsg_t *reply = zmsg_new();
//** Adds UUID + SOURCE to reply message
zframe_t *uuid = zframe_dup(zmsg_first(output));
zframe_t *source = zframe_dup(zmsg_next(output));
zmsg_add(reply, uuid);
zmsg_add(reply, source);
//** Sends reply to broker
rrwrk_send_to_broker(self, RRWRK_REPLY, reply);
//** Sends output to sinker
//zmsg_send(&output, self->sender);
rrwrk_send_to_sinker(self, RRWRK_OUTPUT, output);
zmsg_destroy(&output);
zmsg_destroy(&reply);
}
//** Dispatch task if any
while (true) {
zmq_pollitem_t pipe_write = {self->pipe, 0, ZMQ_POLLOUT, 0};
zmq_poll(&pipe_write, 1, 0);
if ((pipe_write.revents & ZMQ_POLLOUT) && (zlist_size(self->data))) {
zmsg_t* data = (zmsg_t *)zlist_pop(self->data);
zmsg_send(&data, self->pipe);
printf("Dispatched one task.\n");
} else
break;
}
//** Send HEARTBEAT if it's time
if (zclock_time() > self->heartbeat_at) {
rrwrk_print(self);
rrwrk_send_to_broker(self, RRWRK_HEARTBEAT, NULL);
self->heartbeat_at = zclock_time() + self->heartbeat;
}
}
if (zctx_interrupted)
log_printf(0, "W: interrupt received. Killing worker...\n");
return -1;
}
int main(int argc, char** argv) {
long mashine_number = -1;
char *bind_address = NULL;
long port = -1;
int opt;
char *endptr;
/* parse command line options */
while ((opt = getopt(argc, argv, "m:b:p:h")) != -1) {
switch (opt) {
case 'm':
errno = 0;
mashine_number = strtol(optarg, &endptr, 10);
if ((errno != 0 && mashine_number == 0) ||
(*endptr != '\0') ||
(mashine_number < 0 || mashine_number > 15)) {
fprintf(stderr, "invalid mashine number\n");
return EXIT_FAILURE;
}
break;
case 'b':
bind_address = strndup(optarg, 45);
break;
case 'p':
errno = 0;
port = strtol(optarg, &endptr, 10);
if ((errno != 0) ||
(*endptr != '\0') ||
(port <= 0 || port > 65535)) {
fprintf(stderr, "invalid port number\n");
return EXIT_FAILURE;
}
break;
case 'h':
print_usage(stdout);
return EXIT_SUCCESS;
case '?':
print_usage(stderr);
return EXIT_FAILURE;
default:
return EXIT_FAILURE;
}
}
if (mashine_number == -1) {
print_usage(stderr);
return EXIT_FAILURE;
}
if (bind_address == NULL) {
fprintf(stdout, "No bind address specified. Using default address: %s\n", BIND_ADDRESS);
bind_address = BIND_ADDRESS;
}
if (port == -1) {
fprintf(stdout, "No port number specified. Using default port number: %d\n", PORT);
port = PORT;
}
/* create context object */
zctx_t *ctx = zctx_new();
/* create socket objects */
void *server = zsocket_new(ctx, ZMQ_ROUTER);
void *dispatcher = zsocket_new(ctx, ZMQ_DEALER);
/* bind server/dispatcher socket */
zsocket_bind(server, "tcp://%s:%ld", bind_address, port);
zsocket_bind(dispatcher, "inproc://zid");
/* create worker threads */
setting_t *s;
int i;
for (i = 0; i < NUMBER_OF_WORKERS; i++) {
s = (setting_t *) malloc(sizeof(setting_t));
s->mashine_number = mashine_number;
s->thread_number = i;
zthread_fork(ctx, worker, s);
}
/* zmq_proxy runs in current thread */
zmq_proxy(server, dispatcher, NULL);
zctx_destroy (&ctx);
return EXIT_SUCCESS;
}
/* Register an specific sm_io modules to this device */
devio_err_e devio_register_sm (devio_t *self, uint32_t smio_id, void *priv)
{
assert (self);
/* Search the sm_io_mod_dsapatch table for the smio_id and,
* if found, call the correspondent bootstrap code to initilize
* the sm_io module */
th_boot_args_t *th_args = NULL;
/* For now, just do a simple linear search. We can afford this, as
* we don't expect to insert new sm_io modules often */
unsigned int i;
DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE,
"[dev_io_core:register_sm] smio_mod_dispatch table size = %ld\n",
ARRAY_SIZE(smio_mod_dispatch));
for (i = 0; i < ARRAY_SIZE(smio_mod_dispatch); ++i) {
if (smio_mod_dispatch[i].id == smio_id) {
/* Found! Call bootstrap code and insert in
* hash table */
/* FIXME: Why do I need this? smio always gets initilized
* after smio_mod_dispatch[i].bootstrap_ops->smio_boot (self); */
/* smio_t *smio = NULL; */
/* It is expected tha after the boot () call the operations
* this sm_io inscate can handle are already registered! */
DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE,
"[dev_io_core:register_sm] Allocating thread args\n");
/* Alloacate thread arguments struct and pass it to the
* thread. It is the responsability of the calling thread
* to clear this structure after using it! */
th_boot_args_t *th_args = zmalloc (sizeof *th_args);
ASSERT_ALLOC (th_args, err_th_args_alloc);
th_args->parent = self;
/* FIXME: weak identifier */
th_args->smio_id = i;
th_args->broker = self->endpoint_broker;
th_args->service = self->name;
th_args->verbose = self->verbose;
th_args->priv = priv;
DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE,
"[dev_io_core:register_sm] Calling boot func\n");
uint32_t pipe_idx = self->nnodes++;
self->pipes [pipe_idx] = zthread_fork (self->ctx, smio_startup,
th_args);
/* self->pipes [pipe_idx] = zthread_fork (self->ctx,
smio_mod_dispatch[i].bootstrap_ops->thread_boot, th_args); */
/*smio = smio_mod_dispatch[i].bootstrap_ops->boot (self);*/
/*ASSERT_ALLOC (smio, err_smio_alloc); */
/* Stringify ID */
DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE,
"[dev_io_core:register_sm] Stringify hash ID\n");
char *key = halutils_stringify_key (smio_mod_dispatch[i].id);
ASSERT_ALLOC (key, err_key_alloc);
DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE,
"[dev_io_core:register_sm] Inserting hash with key: %s\n", key);
zhash_insert (self->sm_io_h, key, self->pipes [pipe_idx]);
free (key);
/* stop on first match */
break;
}
}
//free (th_args);
return DEVIO_SUCCESS;
err_key_alloc:
free (th_args);
err_th_args_alloc:
return DEVIO_ERR_ALLOC;
}
void
zsync_node_engine (void *args, zctx_t *ctx, void *pipe)
{
int rc;
zsync_node_t *self = zsync_node_new ();
self->ctx = ctx;
self->zyre = zyre_new (ctx);
self->zsync_pipe = pipe;
// Join group
rc = zyre_join (self->zyre, "ZSYNC");
assert (rc == 0);
// Give time to interconnect
zclock_sleep (250);
zpoller_t *poller = zpoller_new (zyre_socket (self->zyre), self->zsync_pipe, NULL);
// Create thread for file management
self->file_pipe = zthread_fork (self->ctx, zsync_ftmanager_engine, NULL);
zpoller_add (poller, self->file_pipe);
// Create thread for credit management
self->credit_pipe = zthread_fork (self->ctx, zsync_credit_manager_engine, NULL);
zpoller_add (poller, self->credit_pipe);
// Start receiving messages
printf("[ND] started\n");
while (!zpoller_terminated (poller)) {
void *which = zpoller_wait (poller, -1);
if (which == zyre_socket (self->zyre)) {
zsync_node_recv_from_zyre (self);
}
else
if (which == self->zsync_pipe) {
printf("[ND] Recv Agent\n");
zsync_node_recv_from_agent (self);
}
else
if (which == self->file_pipe) {
printf("[ND] Recv FT Manager\n");
zsync_ftm_msg_t *msg = zsync_ftm_msg_recv (self->file_pipe);
char *receiver = zsync_ftm_msg_receiver (msg);
char *zyre_uuid = zsync_node_zyre_uuid (self, receiver);
if (zyre_uuid) {
char *path = zsync_ftm_msg_path (msg);
uint64_t sequence = zsync_ftm_msg_sequence (msg);
uint64_t chunk_size = zsync_ftm_msg_chunk_size (msg);
uint64_t offset = zsync_ftm_msg_offset (msg);
zsync_msg_send_req_chunk (pipe, path, chunk_size, offset);
zsync_msg_t *zsmsg = zsync_msg_recv (pipe);
zchunk_t *chunk = zsync_msg_chunk (zsmsg);
zframe_t *frame = zframe_new (zchunk_data (chunk), zchunk_size (chunk));
zmsg_t *zmsg = zmsg_new ();
zs_msg_pack_chunk (zmsg, sequence, path, offset, frame);
zyre_whisper (self->zyre, zyre_uuid, &zmsg);
zsync_ftm_msg_destroy (&msg);
zsync_msg_destroy (&zsmsg);
}
}
else
if (which == self->credit_pipe) {
printf("[ND] Recv Credit Manager\n");
zsync_credit_msg_t *cmsg = zsync_credit_msg_recv (self->credit_pipe);
char *receiver = zsync_credit_msg_receiver (cmsg);
char *zyre_uuid = zsync_node_zyre_uuid (self, receiver);
if (zyre_uuid) {
zmsg_t *credit_msg = zsync_credit_msg_credit (cmsg);
assert (rc == 0);
zyre_whisper (self->zyre, zyre_uuid, &credit_msg);
}
zsync_credit_msg_destroy (&cmsg);
}
if (self->terminated) {
break;
}
}
zpoller_destroy (&poller);
zsync_node_destroy (&self);
printf("[ND] stopped\n");
}
