int rrwrk_start(rrwrk_t *self, wrk_task_fn *cb) { self->heartbeat_at = zclock_time() + self->heartbeat; self->cb = cb; //** Start task thread and wait for synchronization signal self->pipe = zthread_fork(self->ctx, rrtask_manager_fn, (void *)self); assert(self->pipe); free(zstr_recv(self->pipe)); //self->liveness = HEARTBEAT_LIVENESS; //** Don't do reconnect before the first connection established while(!zctx_interrupted) { zmq_pollitem_t items[] = {{self->worker, 0, ZMQ_POLLIN, 0}, {self->pipe, 0, ZMQ_POLLIN, 0}}; //** Be aware: this must be within while loop!! int rc = zmq_poll(items, 2, self->heartbeat * ZMQ_POLL_MSEC); if (rc == -1) break; //** Interrupted if (items[0].revents & ZMQ_POLLIN) { //** Data from broker is ready zmsg_t *msg = zmsg_recv(self->worker); if (!msg) break; //** Interrupted. Need to do more research to confirm it self->liveness = HEARTBEAT_LIVENESS; self->last_heartbeat = zclock_time(); //** Dont try to handle errors, just assert noisily assert(zmsg_size(msg) >= 3); //** empty + header + command + ... zframe_t *empty = zmsg_pop(msg); assert(zframe_streq(empty, "")); zframe_destroy(&empty); zframe_t *header = zmsg_pop(msg); assert(zframe_streq(header, RR_WORKER)); zframe_destroy(&header); zframe_t *command = zmsg_pop(msg); if (zframe_streq(command, RRWRK_REQUEST)) { assert(zmsg_size(msg) == 3); //** UUID + SOURCE + INPUT DATA self->total_received++; zmq_pollitem_t item = {self->pipe, 0, ZMQ_POLLOUT, 0}; int rc = zmq_poll(&item, 1, 0); assert(rc != -1); if (item.revents & ZMQ_POLLOUT) { //** Dispatch it if worker is ready //** Send task to task manager zmsg_send(&msg, self->pipe); } else { //** Otherwise put it on waiting list zlist_push(self->data, zmsg_dup(msg)); } } else if (zframe_streq(command, RRWRK_HEARTBEAT)) { ; //** Do nothing for heartbeat } else if (zframe_streq(command, RRWRK_DISCONNECT)) { rrwrk_connect_to_broker(self); } else { log_printf(0, "E: invalid input message\n"); } zframe_destroy(&command); zmsg_destroy(&msg); } else if ((zclock_time() - self->heartbeat) > self->last_heartbeat) { if(--self->liveness == 0) { rrwrk_print(self); log_printf(0, "W: Disconnected from broker - retrying ...\n"); rrwrk_print(self); zclock_sleep(self->reconnect); rrwrk_connect_to_broker(self); } } if (items[1].revents & ZMQ_POLLIN) { //** Data from pipe is ready zmsg_t *output = zmsg_recv(self->pipe); assert(zmsg_size(output) == 3); //** UUID + SOURCE + DATA self->total_finished++; zmsg_t *reply = zmsg_new(); //** Adds UUID + SOURCE to reply message zframe_t *uuid = zframe_dup(zmsg_first(output)); zframe_t *source = zframe_dup(zmsg_next(output)); zmsg_add(reply, uuid); zmsg_add(reply, source); //** Sends reply to broker rrwrk_send_to_broker(self, RRWRK_REPLY, reply); //** Sends output to sinker //zmsg_send(&output, self->sender); rrwrk_send_to_sinker(self, RRWRK_OUTPUT, output); zmsg_destroy(&output); zmsg_destroy(&reply); } //** Dispatch task if any while (true) { zmq_pollitem_t pipe_write = {self->pipe, 0, ZMQ_POLLOUT, 0}; zmq_poll(&pipe_write, 1, 0); if ((pipe_write.revents & ZMQ_POLLOUT) && (zlist_size(self->data))) { zmsg_t* data = (zmsg_t *)zlist_pop(self->data); zmsg_send(&data, self->pipe); printf("Dispatched one task.\n"); } else break; } //** Send HEARTBEAT if it's time if (zclock_time() > self->heartbeat_at) { rrwrk_print(self); rrwrk_send_to_broker(self, RRWRK_HEARTBEAT, NULL); self->heartbeat_at = zclock_time() + self->heartbeat; } } if (zctx_interrupted) log_printf(0, "W: interrupt received. Killing worker...\n"); return -1; }
int main(int argc, char** argv) { long mashine_number = -1; char *bind_address = NULL; long port = -1; int opt; char *endptr; /* parse command line options */ while ((opt = getopt(argc, argv, "m:b:p:h")) != -1) { switch (opt) { case 'm': errno = 0; mashine_number = strtol(optarg, &endptr, 10); if ((errno != 0 && mashine_number == 0) || (*endptr != '\0') || (mashine_number < 0 || mashine_number > 15)) { fprintf(stderr, "invalid mashine number\n"); return EXIT_FAILURE; } break; case 'b': bind_address = strndup(optarg, 45); break; case 'p': errno = 0; port = strtol(optarg, &endptr, 10); if ((errno != 0) || (*endptr != '\0') || (port <= 0 || port > 65535)) { fprintf(stderr, "invalid port number\n"); return EXIT_FAILURE; } break; case 'h': print_usage(stdout); return EXIT_SUCCESS; case '?': print_usage(stderr); return EXIT_FAILURE; default: return EXIT_FAILURE; } } if (mashine_number == -1) { print_usage(stderr); return EXIT_FAILURE; } if (bind_address == NULL) { fprintf(stdout, "No bind address specified. Using default address: %s\n", BIND_ADDRESS); bind_address = BIND_ADDRESS; } if (port == -1) { fprintf(stdout, "No port number specified. Using default port number: %d\n", PORT); port = PORT; } /* create context object */ zctx_t *ctx = zctx_new(); /* create socket objects */ void *server = zsocket_new(ctx, ZMQ_ROUTER); void *dispatcher = zsocket_new(ctx, ZMQ_DEALER); /* bind server/dispatcher socket */ zsocket_bind(server, "tcp://%s:%ld", bind_address, port); zsocket_bind(dispatcher, "inproc://zid"); /* create worker threads */ setting_t *s; int i; for (i = 0; i < NUMBER_OF_WORKERS; i++) { s = (setting_t *) malloc(sizeof(setting_t)); s->mashine_number = mashine_number; s->thread_number = i; zthread_fork(ctx, worker, s); } /* zmq_proxy runs in current thread */ zmq_proxy(server, dispatcher, NULL); zctx_destroy (&ctx); return EXIT_SUCCESS; }
/* Register an specific sm_io modules to this device */ devio_err_e devio_register_sm (devio_t *self, uint32_t smio_id, void *priv) { assert (self); /* Search the sm_io_mod_dsapatch table for the smio_id and, * if found, call the correspondent bootstrap code to initilize * the sm_io module */ th_boot_args_t *th_args = NULL; /* For now, just do a simple linear search. We can afford this, as * we don't expect to insert new sm_io modules often */ unsigned int i; DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE, "[dev_io_core:register_sm] smio_mod_dispatch table size = %ld\n", ARRAY_SIZE(smio_mod_dispatch)); for (i = 0; i < ARRAY_SIZE(smio_mod_dispatch); ++i) { if (smio_mod_dispatch[i].id == smio_id) { /* Found! Call bootstrap code and insert in * hash table */ /* FIXME: Why do I need this? smio always gets initilized * after smio_mod_dispatch[i].bootstrap_ops->smio_boot (self); */ /* smio_t *smio = NULL; */ /* It is expected tha after the boot () call the operations * this sm_io inscate can handle are already registered! */ DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE, "[dev_io_core:register_sm] Allocating thread args\n"); /* Alloacate thread arguments struct and pass it to the * thread. It is the responsability of the calling thread * to clear this structure after using it! */ th_boot_args_t *th_args = zmalloc (sizeof *th_args); ASSERT_ALLOC (th_args, err_th_args_alloc); th_args->parent = self; /* FIXME: weak identifier */ th_args->smio_id = i; th_args->broker = self->endpoint_broker; th_args->service = self->name; th_args->verbose = self->verbose; th_args->priv = priv; DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE, "[dev_io_core:register_sm] Calling boot func\n"); uint32_t pipe_idx = self->nnodes++; self->pipes [pipe_idx] = zthread_fork (self->ctx, smio_startup, th_args); /* self->pipes [pipe_idx] = zthread_fork (self->ctx, smio_mod_dispatch[i].bootstrap_ops->thread_boot, th_args); */ /*smio = smio_mod_dispatch[i].bootstrap_ops->boot (self);*/ /*ASSERT_ALLOC (smio, err_smio_alloc); */ /* Stringify ID */ DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE, "[dev_io_core:register_sm] Stringify hash ID\n"); char *key = halutils_stringify_key (smio_mod_dispatch[i].id); ASSERT_ALLOC (key, err_key_alloc); DBE_DEBUG (DBG_DEV_IO | DBG_LVL_TRACE, "[dev_io_core:register_sm] Inserting hash with key: %s\n", key); zhash_insert (self->sm_io_h, key, self->pipes [pipe_idx]); free (key); /* stop on first match */ break; } } //free (th_args); return DEVIO_SUCCESS; err_key_alloc: free (th_args); err_th_args_alloc: return DEVIO_ERR_ALLOC; }
void zsync_node_engine (void *args, zctx_t *ctx, void *pipe) { int rc; zsync_node_t *self = zsync_node_new (); self->ctx = ctx; self->zyre = zyre_new (ctx); self->zsync_pipe = pipe; // Join group rc = zyre_join (self->zyre, "ZSYNC"); assert (rc == 0); // Give time to interconnect zclock_sleep (250); zpoller_t *poller = zpoller_new (zyre_socket (self->zyre), self->zsync_pipe, NULL); // Create thread for file management self->file_pipe = zthread_fork (self->ctx, zsync_ftmanager_engine, NULL); zpoller_add (poller, self->file_pipe); // Create thread for credit management self->credit_pipe = zthread_fork (self->ctx, zsync_credit_manager_engine, NULL); zpoller_add (poller, self->credit_pipe); // Start receiving messages printf("[ND] started\n"); while (!zpoller_terminated (poller)) { void *which = zpoller_wait (poller, -1); if (which == zyre_socket (self->zyre)) { zsync_node_recv_from_zyre (self); } else if (which == self->zsync_pipe) { printf("[ND] Recv Agent\n"); zsync_node_recv_from_agent (self); } else if (which == self->file_pipe) { printf("[ND] Recv FT Manager\n"); zsync_ftm_msg_t *msg = zsync_ftm_msg_recv (self->file_pipe); char *receiver = zsync_ftm_msg_receiver (msg); char *zyre_uuid = zsync_node_zyre_uuid (self, receiver); if (zyre_uuid) { char *path = zsync_ftm_msg_path (msg); uint64_t sequence = zsync_ftm_msg_sequence (msg); uint64_t chunk_size = zsync_ftm_msg_chunk_size (msg); uint64_t offset = zsync_ftm_msg_offset (msg); zsync_msg_send_req_chunk (pipe, path, chunk_size, offset); zsync_msg_t *zsmsg = zsync_msg_recv (pipe); zchunk_t *chunk = zsync_msg_chunk (zsmsg); zframe_t *frame = zframe_new (zchunk_data (chunk), zchunk_size (chunk)); zmsg_t *zmsg = zmsg_new (); zs_msg_pack_chunk (zmsg, sequence, path, offset, frame); zyre_whisper (self->zyre, zyre_uuid, &zmsg); zsync_ftm_msg_destroy (&msg); zsync_msg_destroy (&zsmsg); } } else if (which == self->credit_pipe) { printf("[ND] Recv Credit Manager\n"); zsync_credit_msg_t *cmsg = zsync_credit_msg_recv (self->credit_pipe); char *receiver = zsync_credit_msg_receiver (cmsg); char *zyre_uuid = zsync_node_zyre_uuid (self, receiver); if (zyre_uuid) { zmsg_t *credit_msg = zsync_credit_msg_credit (cmsg); assert (rc == 0); zyre_whisper (self->zyre, zyre_uuid, &credit_msg); } zsync_credit_msg_destroy (&cmsg); } if (self->terminated) { break; } } zpoller_destroy (&poller); zsync_node_destroy (&self); printf("[ND] stopped\n"); }