/* Accumulate worker messages with statistics */
static void add_stat_message(chunk_ptr msg) {
size_t *stat_summary = NULL;
stat_messages[stat_message_cnt++] = msg;
/* See if we've accumulated a full set */
if (stat_message_cnt >= worker_cnt) {
size_t nstat = stat_messages[0]->length - 1;
if (flush_requestor_fd >= 0)
stat_summary = calloc_or_fail(nstat * 3, sizeof(size_t),
"add_stat_message");
/* Accumulate and print */
#if RPT >= 1
report(1, "Worker statistics:");
#endif
size_t i, w;
for (i = 0; i < nstat; i++) {
chunk_ptr msg = stat_messages[0];
word_t minval, maxval, sumval;
minval = maxval = sumval = chunk_get_word(msg, i+1);
for (w = 1; w < worker_cnt; w++) {
chunk_ptr msg = stat_messages[w];
word_t val = chunk_get_word(msg, i+1);
if (val < minval)
minval = val;
if (val > maxval)
maxval = val;
sumval += val;
}
if (stat_summary) {
stat_summary[3*i + 0] = minval;
stat_summary[3*i + 1] = maxval;
stat_summary[3*i + 2] = sumval;
}
#if RPT >= 1
report(1,
"Parameter %d\tMin: %" PRIu64 "\tMax: %" PRIu64 "\tAvg: %.2f\tSum: %" PRIu64,
(int) i, minval, maxval, (double) sumval/worker_cnt, sumval);
#endif
}
if (flush_requestor_fd >= 0) {
chunk_ptr msg = msg_new_stat(worker_cnt, nstat*3, stat_summary);
if (chunk_write(flush_requestor_fd, msg)) {
#if RPT >= 5
report(5, "Sent statistical summary to client at fd %d",
flush_requestor_fd);
#endif
} else {
err(false, "Failed to send statistical summary to client at fd %d",
flush_requestor_fd);
}
chunk_free(msg);
free_array(stat_summary, nstat*3, sizeof(size_t));
}
for (w = 0; w < worker_cnt; w++) {
chunk_free(stat_messages[w]);
stat_messages[w] = NULL;
}
stat_message_cnt = 0;
flush_requestor_fd = -1;
}
}
/* Insert an operand into an operation */
void op_insert_operand(chunk_ptr op, chunk_ptr oper, unsigned offset) {
size_t i;
size_t n = oper->length-OPER_HEADER_CNT;
#if RPT >= 5
dword_t dh = chunk_get_dword(op, 0);
unsigned opcode = msg_get_dheader_opcode(dh);
word_t vmask = chunk_get_word(op, 2);
report(5,
"Inserting operand with %u words into op with opcode %u at offset %u. Mask 0x%lx",
(unsigned) n, opcode, offset, vmask);
#endif
for (i = 0; i < n; i++) {
word_t w = chunk_get_word(oper, i+OPER_HEADER_CNT);
op_insert_word(op, w, i+offset);
}
}
/* Extract subchunk */
chunk_ptr chunk_get_chunk(chunk_ptr cp, size_t offset, size_t length) {
chunk_ptr ncp = chunk_new(length);
size_t i;
for (i = 0; i < length; i++) {
chunk_insert_word(ncp, chunk_get_word(cp, i+offset), i);
}
return ncp;
}
/* Get string that has been stored as chunk */
char * chunk2str(chunk_ptr cp) {
char buf[WORD_BYTES * CHUNK_MAX_LENGTH+1];
/* Make sure string is terminated */
buf[WORD_BYTES * cp->length] = '\0';
word_t *wp = (word_t *) buf;
size_t cidx;
for (cidx = 0; cidx < cp->length; cidx++) {
wp[cidx] = chunk_get_word(cp, cidx);
}
return strsave_or_fail(buf, "chunk2str");
}
/* Check whether all fields an an operator are valid */
bool op_check_full(chunk_ptr op) {
if (op == NULL) {
err(false, "op_check_full given null pointer");
return false;
}
word_t len = op->length;
word_t vmask = chunk_get_word(op, 2);
/* Create checking vmask */
word_t cmask = len == OP_MAX_LENGTH ? ~0ULL : (1ULL << len) - 1;
return vmask == cmask;
}
/* Insert word into operator, updating its valid mask.
Offset includes header size */
void op_insert_word(chunk_ptr op, word_t wd, size_t offset) {
word_t vmask = chunk_get_word(op, 2);
word_t nvmask;
word_t idx = (word_t) 1 << offset;
if (vmask & idx) {
err(false,
"Inserting into already filled position in operator. Offset = %lu", offset);
}
chunk_insert_word(op, wd, offset);
nvmask = vmask | idx;
chunk_insert_word(op, nvmask, 2);
#if RPT >= 6
dword_t dh = chunk_get_dword(op, 0);
word_t id = msg_get_dheader_op_id(dh);
report(6,
"Inserted word, offset %d, operation with id 0x%lx.\n"
" Total size = %d. Vmask 0x%lx --> 0x%lx",
(int) offset, id, op->length, vmask, nvmask);
#endif
}
void run_worker() {
while (true) {
/* Select among controller port, and connections to routers */
FD_ZERO(&cset);
maxcfd = 0;
add_cfd(controller_fd);
unsigned ridx;
for (ridx = 0; ridx < nrouters; ridx++)
add_cfd(router_fd_array[ridx]);
buf_select(maxcfd+1, &cset, NULL, NULL, NULL);
int fd;
for (fd = 0; fd <= maxcfd; fd++) {
if (!FD_ISSET(fd, &cset))
continue;
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (fd == controller_fd) {
err(true, "Unexpected EOF from controller (fatal)");
} else {
err(false,
"Unexpected EOF from router with fd %d (shutting down)", fd);
finish_cmd();
exit(0);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
/* Rely on fact that following message fields in same location
for both single and double-word headers */
unsigned code = msg_get_header_code(h);
unsigned agent = msg_get_header_agent(h);
unsigned opcode = msg_get_header_opcode(h);
if (fd == controller_fd) {
/* Message from controller */
switch(code) {
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1, "Received kill message from controller");
#endif
quit_agent(0, NULL);
return;
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 5
report(5, "Received flush message from controller");
#endif
if (flush_helper) {
chunk_ptr msg = flush_helper();
if (!msg)
break;
if (chunk_write(controller_fd, msg)) {
#if RPT >= 5
report(5,
"Sent statistics information to controller");
#endif
} else {
err(false,
"Failed to send statistics information to controller");
}
chunk_free(msg);
}
break;
case MSG_CLIOP_DATA:
report(5,
"Received client operation data. Agent = %u", agent);
word_t *data = &msg->words[1];
unsigned nword = msg->length - 1;
if (gop_start_helper)
gop_start_helper(agent, opcode, nword, data);
chunk_free(msg);
chunk_ptr rmsg = msg_new_cliop_ack(agent);
if (chunk_write(controller_fd, rmsg)) {
#if RPT >= 5
report(5,
"Acknowledged client operation data. Agent = %u",
agent);
#endif
} else {
err(false,
"Failed to acknowledge client operation data. Agent = %u",
agent);
}
chunk_free(rmsg);
break;
case MSG_CLIOP_ACK:
#if RPT >= 5
report(5, "Received client operation ack. Agent = %u", agent);
#endif
if (gop_finish_helper)
gop_finish_helper(agent);
chunk_free(msg);
break;
case MSG_GC_START:
chunk_free(msg);
gc_start(code);
break;
case MSG_GC_FINISH:
chunk_free(msg);
gc_finish(code);
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)", code);
}
} else {
/* Must be message from router */
switch (code) {
case MSG_OPERATION:
receive_operation(msg);
break;
case MSG_OPERAND:
receive_operand(msg);
break;
default:
chunk_free(msg);
err(false,
"Received message with unknown code %u (ignored)", code);
}
}
}
}
quit_agent(0, NULL);
}
/*
Initiate global operation from client.
Returns to client when all workers ready to perform their operations
*/
bool start_client_global(unsigned opcode, unsigned nword, word_t *data) {
chunk_ptr rmsg = msg_new_cliop_data(own_agent, opcode, nword, data);
if (!chunk_write(controller_fd, rmsg)) {
err(false, "Could not send client operation message to controller");
chunk_free(rmsg);
return false;
}
chunk_free(rmsg);
/* Read ACK, as well as any other messages the controller might have */
bool done = false;
bool ok = true;
while (!done) {
bool eof = false;
chunk_ptr msg = chunk_read_unbuffered(controller_fd, &eof);
if (eof) {
/* Unexpected EOF */
err(true, "Unexpected EOF from controller (fatal)");
close(controller_fd);
done = true; ok = false;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
switch (code) {
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 5
report(5,
"Received flush message from controller, superceding client global operation");
#endif
if (flush_helper) {
flush_helper();
}
done = true; ok = false;
break;
case MSG_STAT:
#if RPT >= 5
report(5, "Received summary statistics from controller");
#endif
if (stat_helper) {
/* Get a copy of the byte usage from memory allocator */
stat_helper(msg);
}
chunk_free(msg);
break;
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1,
"Received kill message from controller, superceding client global operation");
#endif
finish_cmd();
done = true; ok = false;
break;
case MSG_CLIOP_ACK:
chunk_free(msg);
#if RPT >= 5
report(5,
"Received acknowledgement for client global operation");
#endif
done = true; ok = true;
break;
case MSG_GC_START:
/* Got notice that should initiate garbage collection.
Defer until current operation done */
#if RPT >= 3
report(3, "Deferring GC start");
#endif
chunk_free(msg);
gc_state = GC_DEFER;
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)", code);
}
}
return ok;
}
void init_agent(bool iscli, char *controller_name, unsigned controller_port,
bool try_self_route, bool try_local_router) {
operator_table = word_keyvalue_new();
deferred_operand_table = word_keyvalue_new();
size_t i;
for (i = 0; i < NSTATA; i++)
agent_stat_counter[i] = 0;
chunk_ptr msg;
bool eof;
isclient = iscli;
self_route = try_self_route;
controller_fd = open_clientfd(controller_name, controller_port);
if (controller_fd < 0)
err(true,
"Cannot create connection to controller at %s:%u",
controller_name, controller_port);
else {
#if RPT >= 2
report(2, "Connection to controller has descriptor %d", controller_fd);
#endif
}
msg = isclient ? msg_new_register_client() : msg_new_register_worker();
bool sok = chunk_write(controller_fd, msg);
#if RPT >= 3
report(3, "Sent %s registration to controller",
isclient ? "client" : "worker");
#endif
chunk_free(msg);
if (!sok)
err(true, "Could not send registration message to controller");
/* Get acknowledgement from controller */
bool first = true;
/* Anticipated number of routers
(Will be updated when get first message from controller) */
nrouters = 1;
chunk_ptr amsg = NULL;
unsigned ridx = 0;;
while (ridx < nrouters) {
msg = chunk_read_unbuffered(controller_fd, &eof);
if (eof) {
err(true,
"Unexpected EOF from controller while getting router map");
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
switch (code) {
case MSG_ACK_AGENT:
if (first) {
own_agent = msg_get_header_agent(h);
amsg = msg_new_register_agent(own_agent);
nworkers = msg_get_header_workercount(h);
nrouters = msg_get_header_wordcount(h);
router_fd_array = calloc_or_fail(nrouters, sizeof(int),
"init_agent");
#if RPT >= 3
report(3,
"Ack from controller. Agent Id %u. %d workers. %d routers.",
own_agent, nworkers, nrouters);
#endif
first = false;
}
int i;
for (i = 1; i < msg->length; i++) {
word_t h = chunk_get_word(msg, i);
int fd;
unsigned ip = msg_get_header_ip(h);
unsigned port = msg_get_header_port(h);
#if RPT >= 4
report(4, "Attempting to add router %u with ip 0x%x, port %d",
ridx, ip, port);
#endif
fd = open_clientfd_ip(ip, port);
if (fd < 0) {
err(true, "Couldn't add router with ip 0x%x, port %d",
ip, port);
} else {
router_fd_array[ridx++] = fd;
#if RPT >= 3
report(3, "Added router %u with ip 0x%x, port %d, fd %d",
ridx, ip, port, fd);
#endif
if (!chunk_write(fd, amsg)) {
err(true,
"Couldn't send agent registration message to router with ip 0x%x, port %u",
ip, port);
}
if (try_local_router && local_router_fd == -1 &&
match_self_ip(ip))
{
local_router_fd = fd;
#if RPT >= 5
report(5,
"Router with fd %d designated as local router and prioritized for sending packets",
fd);
#endif
}
}
}
chunk_free(msg);
break;
case MSG_NACK:
err(true, "Connection request refused.");
break;
default:
err(false,
"Unexpected message code %u while getting router information", code);
chunk_free(msg);
break;
}
}
chunk_free(amsg);
#if RPT >= 2
report(2, "All %d routers connected", nrouters);
#endif
if (isclient) {
add_quit_helper(quit_agent);
add_cmd("kill", do_agent_kill,
" | Shutdown system");
add_cmd("flush", do_agent_flush,
" | Flush system");
add_cmd("collect", do_agent_gc,
" | Initiate garbage collection");
} else {
/* Worker must notify controller that it's ready */
chunk_ptr rmsg = msg_new_worker_ready(own_agent);
if (chunk_write(controller_fd, rmsg)) {
#if RPT >= 3
report(3, "Notified controller that worker is ready");
#endif
} else {
err(true, "Couldn't notify controller that worker is ready");
}
chunk_free(rmsg);
}
}
/* Client command loop only considers console inputs + controller messages */
void run_client(char *infile_name) {
if (!start_cmd(infile_name))
return;
while (!cmd_done()) {
/* Select among controller port, and connections to routers */
FD_ZERO(&cset);
maxcfd = 0;
add_cfd(controller_fd);
cmd_select(maxcfd+1, &cset, NULL, NULL, NULL);
if (cmd_done())
break;
int fd;
for (fd = 0; fd <= maxcfd; fd++) {
if (!FD_ISSET(fd, &cset))
continue;
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (fd == controller_fd) {
err(true, "Unexpected EOF from controller (fatal)");
} else {
err(false,
"Unexpected EOF from unexpected source. fd %d (ignored)", fd);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
if (fd == controller_fd) {
/* Message from controller */
switch(code) {
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 5
report(5, "Received flush message from controller");
#endif
if (flush_helper) {
flush_helper();
}
break;
case MSG_STAT:
#if RPT >= 5
report(5, "Received summary statistics from controller");
#endif
if (stat_helper) {
/* Get a copy of the byte usage from mem. allocator */
stat_helper(msg);
}
chunk_free(msg);
/* Client can proceed with next command */
unblock_console();
break;
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1, "Received kill message from controller");
#endif
finish_cmd();
break;
case MSG_GC_START:
chunk_free(msg);
gc_start(code);
break;
case MSG_GC_FINISH:
chunk_free(msg);
gc_finish(code);
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)",
code);
}
} else {
err(false,
"Received message from unknown source. fd %d (ignored)",
fd);
close(fd);
continue;
}
}
}
}
/* Fire an operation and wait for returned operand */
chunk_ptr fire_and_wait_defer(chunk_ptr msg) {
chunk_ptr rval = NULL;
if (!send_op(msg)) {
err(false, "Failed to send message");
return NULL;
}
bool local_done = false;
while (!(local_done || cmd_done())) {
/* Select among controller port, and connections to routers.
Do not accept console input */
FD_ZERO(&rset);
maxrfd = 0;
add_rfd(controller_fd);
unsigned ridx;
for (ridx = 0; ridx < nrouters; ridx++)
add_rfd(router_fd_array[ridx]);
buf_select(maxrfd+1, &rset, NULL, NULL, NULL);
int fd;
for (fd = 0; fd <= maxrfd; fd++) {
if (!FD_ISSET(fd, &rset))
continue;
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (fd == controller_fd) {
err(true, "Unexpected EOF from controller (fatal)");
} else {
err(false,
"Unexpected EOF from router with fd %d (shutting down)", fd);
finish_cmd();
exit(0);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
if (fd == controller_fd) {
/* Message from controller */
switch(code) {
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1, "Received kill message from controller");
#endif
quit_agent(0, NULL);
break;
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 2
report(2, "Received flush message from controller");
#endif
if (flush_helper) {
flush_helper(0, NULL);
}
break;
case MSG_GC_START:
/* Got notice that should initiate garbage collection.
Defer until current operation done */
#if RPT >= 3
report(3, "Deferring GC start");
#endif
chunk_free(msg);
gc_state = GC_DEFER;
break;
case MSG_GC_FINISH:
#if RPT >= 3
report(3,
"Unexpected GC_FINISH message from controller when waiting for result (ignored)");
#endif
chunk_free(msg);
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)", code);
}
} else {
dword_t dh;
word_t id = 0;
/* Must be message from router */
switch (code) {
case MSG_OPERATION:
chunk_free(msg);
err(false, "Received unexpected operation. Ignored.");
local_done = true;
break;
case MSG_OPERAND:
#if RPT >= 5
dh = chunk_get_dword(msg, 0);
id = msg_get_dheader_op_id(dh);
report(5, "Received operand with id 0x%lx", id);
#endif
rval = msg;
local_done = true;
break;
default:
chunk_free(msg);
err(false,
"Received message with unknown code %u (ignored)", code);
local_done = true;
}
}
}
}
return rval;
}
static void run_controller(char *infile_name) {
if (!start_cmd(infile_name))
return;
while (!cmd_done()) {
FD_ZERO(&set);
int fd;
word_t w;
unsigned ip;
unsigned port;
add_fd(listen_fd);
keyvalue_iterstart(new_conn_map);
/* Check for messages from newly connected clients, workers, and routers */
while (keyvalue_iternext(new_conn_map, &w, NULL)) {
fd = w;
add_fd(fd);
}
if (need_routers == 0) {
/* Accept messages from workers */
set_iterstart(worker_fd_set);
while (set_iternext(worker_fd_set, &w)) {
fd = w;
add_fd(fd);
}
/* Accept messages from clients */
set_iterstart(client_fd_set);
while (set_iternext(client_fd_set, &w)) {
fd = w;
add_fd(fd);
}
}
cmd_select(maxfd+1, &set, NULL, NULL, NULL);
for (fd = 0; fd <= maxfd; fd++) {
if (!FD_ISSET(fd, &set))
continue;
if (fd == listen_fd) {
unsigned ip;
int connfd = accept_connection(fd, &ip);
keyvalue_insert(new_conn_map, (word_t) connfd, (word_t) ip);
#if RPT >= 4
report(4, "Accepted new connection. Connfd = %d, IP = 0x%x",
connfd, ip);
#endif
continue;
}
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (keyvalue_remove(new_conn_map, (word_t) fd, NULL, NULL)) {
err(false, "Unexpected EOF from new connection, fd %d", fd);
} else if (set_member(worker_fd_set, (word_t) fd, true)) {
err(false, "Unexpected EOF from connected worker, fd %d. Shutting down", fd);
/* Shut down system */
finish_cmd();
} else if (set_member(client_fd_set, (word_t) fd, true)) {
#if RPT >= 3
report(3, "Disconnection from client (fd %d)", fd);
#endif
if (need_client_fd_set && set_member(need_client_fd_set,
(word_t) fd, false)) {
#if RPT >= 3
report(3, "Removing client from GC activities");
#endif
handle_gc_msg(MSG_GC_FINISH, 0, fd, true);
}
} else {
err(false, "Unexpected EOF from unknown source, fd %d", fd);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
#if RPT >= 5
report(5, "Received message with code %d from fd %d", code, fd);
#endif
if (keyvalue_remove(new_conn_map, (word_t) fd, NULL, &w)) {
ip = w;
chunk_free(msg);
/* Should be a registration message */
switch (code) {
case MSG_REGISTER_ROUTER:
if (need_routers == 0) {
err(false, "Unexpected router registration. (Ignored)");
close(fd);
break;
}
port = msg_get_header_port(h);
word_t node_id = msg_build_node_id(port, ip);
set_insert(router_addr_set, node_id);
set_insert(router_fd_set, (word_t) fd);
#if RPT >= 4
report(4, "Added router with fd %d. IP 0x%x. Port %u",
fd, ip, port);
#endif
need_routers --;
if (need_routers == 0) {
#if RPT >= 2
report(2, "All routers connected");
#endif
/* Have gotten all of the necessary routers.
Notify any registered workers */
set_iterstart(worker_fd_set);
int wfd;
while (set_iternext(worker_fd_set, &w)) {
wfd = w;
add_agent(wfd, false);
}
}
break;
case MSG_REGISTER_WORKER:
if (worker_fd_set->nelements >= worker_cnt) {
err(false, "Unexpected worker registration. (Ignored)");
close(fd);
break;
}
set_insert(worker_fd_set, (word_t) fd);
#if RPT >= 4
report(4, "Added worker with fd %d", fd);
#endif
if (need_routers == 0)
add_agent(fd, false);
break;
case MSG_REGISTER_CLIENT:
if (gc_state == GC_READY) {
set_insert(client_fd_set, (word_t) fd);
#if RPT >= 4
report(4, "Added client with fd %d", fd);
#endif
if (need_workers == 0)
add_agent(fd, true);
} else {
if (!defer_client_fd_set) {
defer_client_fd_set = word_set_new();
}
set_insert(defer_client_fd_set, (word_t) fd);
#if RPT >= 3
report(3, "Deferring client with fd %d until GC completed",
fd);
#endif
}
break;
default:
err(false, "Unexpected message code %u from new connection",
code);
break;
}
} else if (set_member(worker_fd_set, (word_t) fd, false)) {
/* Message from worker */
switch (code) {
unsigned agent;
unsigned gen;
case MSG_READY_WORKER:
chunk_free(msg);
if (need_workers == 0) {
err(false, "Unexpected worker ready. (Ignored)");
close(fd);
break;
}
need_workers--;
if (need_workers == 0) {
#if RPT >= 2
report(2, "All workers connected");
#endif
/* Notify any pending clients */
set_iterstart(client_fd_set);
int cfd;
while (set_iternext(client_fd_set, &w)) {
cfd = w;
add_agent(cfd, true);
}
}
break;
case MSG_STAT:
/* Message gets stashed away. Don't free it */
add_stat_message(msg);
break;
case MSG_CLIOP_ACK:
/* Worker acknowledging receipt of global operation info */
agent = msg_get_header_agent(h);
int client_fd = receive_global_op_worker_ack(agent);
if (client_fd >= 0) {
/* Have received complete set of acknowledgements. */
/* Send ack to client */
if (chunk_write(client_fd, msg)) {
#if RPT >= 6
report(6,
"Sent ack to client for global operation with id %u", agent);
#endif
} else {
err(false,
"Failed to send ack to client for global operation with id %u. Fd %d",
agent, client_fd);
}
}
chunk_free(msg);
break;
case MSG_GC_START:
case MSG_GC_FINISH:
handle_gc_msg(code, 0, fd, false);
chunk_free(msg);
break;
case MSG_GC_REQUEST:
gen = msg_get_header_generation(h);
chunk_free(msg);
handle_gc_msg(code, gen, fd, false);
break;
default:
chunk_free(msg);
err(false, "Unexpected message code %u from worker", code);
}
} else if (set_member(client_fd_set, (word_t) fd, false)) {
/* Message from client */
switch(code){
unsigned agent;
word_t w;
case MSG_KILL:
/* Shutdown entire system */
chunk_free(msg);
#if RPT >= 2
report(2, "Remote request to kill system");
#endif
finish_cmd();
return;
case MSG_DO_FLUSH:
/* Initiate a flush operation */
chunk_free(msg);
flush_requestor_fd = fd;
do_controller_flush_cmd(0, NULL);
break;
case MSG_CLIOP_DATA:
/* Request for global operation from client */
agent = msg_get_header_agent(h);
add_global_op(agent, fd);
/* Send message to all workers */
set_iterstart(worker_fd_set);
while (set_iternext(worker_fd_set, &w)) {
int worker_fd = (int) w;
if (chunk_write(worker_fd, msg)) {
#if RPT >= 6
report(6,
"Sent global operation information with id %u to worker with fd %d",
agent, worker_fd);
#endif
} else {
err(false,
"Failed to send global operation information with id %u to worker with fd %d",
agent, worker_fd);
}
}
chunk_free(msg);
break;
case MSG_CLIOP_ACK:
/* Completion of global operation by client */
agent = msg_get_header_agent(h);
/* Send message to all workers */
set_iterstart(worker_fd_set);
while (set_iternext(worker_fd_set, &w)) {
int worker_fd = (int) w;
if (chunk_write(worker_fd, msg)) {
#if RPT >= 6
report(6,
"Sent global operation acknowledgement with id %u to worker with fd %d",
agent, worker_fd);
#endif
} else {
err(false,
"Failed to send global operation acknowledgement with id %u to worker with fd %d",
agent, worker_fd);
}
}
chunk_free(msg);
break;
case MSG_GC_START:
case MSG_GC_FINISH:
handle_gc_msg(code, 0, fd, true);
chunk_free(msg);
break;
default:
err(false, "Unexpected message code %u from client", code);
}
} else {
chunk_free(msg);
err(false, "Unexpected message on fd %d (Ignored)", fd);
}
}
}
}