chunk_ptr chunk_read_unbuffered(int fd, bool *eofp)
{
bufferReadBool = 0;
chunk_ptr p = chunk_read(fd, eofp);
bufferReadBool = 1;
return p;
}
void run_worker() {
while (true) {
/* Select among controller port, and connections to routers */
FD_ZERO(&cset);
maxcfd = 0;
add_cfd(controller_fd);
unsigned ridx;
for (ridx = 0; ridx < nrouters; ridx++)
add_cfd(router_fd_array[ridx]);
buf_select(maxcfd+1, &cset, NULL, NULL, NULL);
int fd;
for (fd = 0; fd <= maxcfd; fd++) {
if (!FD_ISSET(fd, &cset))
continue;
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (fd == controller_fd) {
err(true, "Unexpected EOF from controller (fatal)");
} else {
err(false,
"Unexpected EOF from router with fd %d (shutting down)", fd);
finish_cmd();
exit(0);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
/* Rely on fact that following message fields in same location
for both single and double-word headers */
unsigned code = msg_get_header_code(h);
unsigned agent = msg_get_header_agent(h);
unsigned opcode = msg_get_header_opcode(h);
if (fd == controller_fd) {
/* Message from controller */
switch(code) {
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1, "Received kill message from controller");
#endif
quit_agent(0, NULL);
return;
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 5
report(5, "Received flush message from controller");
#endif
if (flush_helper) {
chunk_ptr msg = flush_helper();
if (!msg)
break;
if (chunk_write(controller_fd, msg)) {
#if RPT >= 5
report(5,
"Sent statistics information to controller");
#endif
} else {
err(false,
"Failed to send statistics information to controller");
}
chunk_free(msg);
}
break;
case MSG_CLIOP_DATA:
report(5,
"Received client operation data. Agent = %u", agent);
word_t *data = &msg->words[1];
unsigned nword = msg->length - 1;
if (gop_start_helper)
gop_start_helper(agent, opcode, nword, data);
chunk_free(msg);
chunk_ptr rmsg = msg_new_cliop_ack(agent);
if (chunk_write(controller_fd, rmsg)) {
#if RPT >= 5
report(5,
"Acknowledged client operation data. Agent = %u",
agent);
#endif
} else {
err(false,
"Failed to acknowledge client operation data. Agent = %u",
agent);
}
chunk_free(rmsg);
break;
case MSG_CLIOP_ACK:
#if RPT >= 5
report(5, "Received client operation ack. Agent = %u", agent);
#endif
if (gop_finish_helper)
gop_finish_helper(agent);
chunk_free(msg);
break;
case MSG_GC_START:
chunk_free(msg);
gc_start(code);
break;
case MSG_GC_FINISH:
chunk_free(msg);
gc_finish(code);
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)", code);
}
} else {
/* Must be message from router */
switch (code) {
case MSG_OPERATION:
receive_operation(msg);
break;
case MSG_OPERAND:
receive_operand(msg);
break;
default:
chunk_free(msg);
err(false,
"Received message with unknown code %u (ignored)", code);
}
}
}
}
quit_agent(0, NULL);
}
/* Client command loop only considers console inputs + controller messages */
void run_client(char *infile_name) {
if (!start_cmd(infile_name))
return;
while (!cmd_done()) {
/* Select among controller port, and connections to routers */
FD_ZERO(&cset);
maxcfd = 0;
add_cfd(controller_fd);
cmd_select(maxcfd+1, &cset, NULL, NULL, NULL);
if (cmd_done())
break;
int fd;
for (fd = 0; fd <= maxcfd; fd++) {
if (!FD_ISSET(fd, &cset))
continue;
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (fd == controller_fd) {
err(true, "Unexpected EOF from controller (fatal)");
} else {
err(false,
"Unexpected EOF from unexpected source. fd %d (ignored)", fd);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
if (fd == controller_fd) {
/* Message from controller */
switch(code) {
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 5
report(5, "Received flush message from controller");
#endif
if (flush_helper) {
flush_helper();
}
break;
case MSG_STAT:
#if RPT >= 5
report(5, "Received summary statistics from controller");
#endif
if (stat_helper) {
/* Get a copy of the byte usage from mem. allocator */
stat_helper(msg);
}
chunk_free(msg);
/* Client can proceed with next command */
unblock_console();
break;
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1, "Received kill message from controller");
#endif
finish_cmd();
break;
case MSG_GC_START:
chunk_free(msg);
gc_start(code);
break;
case MSG_GC_FINISH:
chunk_free(msg);
gc_finish(code);
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)",
code);
}
} else {
err(false,
"Received message from unknown source. fd %d (ignored)",
fd);
close(fd);
continue;
}
}
}
}
/* Fire an operation and wait for returned operand */
chunk_ptr fire_and_wait_defer(chunk_ptr msg) {
chunk_ptr rval = NULL;
if (!send_op(msg)) {
err(false, "Failed to send message");
return NULL;
}
bool local_done = false;
while (!(local_done || cmd_done())) {
/* Select among controller port, and connections to routers.
Do not accept console input */
FD_ZERO(&rset);
maxrfd = 0;
add_rfd(controller_fd);
unsigned ridx;
for (ridx = 0; ridx < nrouters; ridx++)
add_rfd(router_fd_array[ridx]);
buf_select(maxrfd+1, &rset, NULL, NULL, NULL);
int fd;
for (fd = 0; fd <= maxrfd; fd++) {
if (!FD_ISSET(fd, &rset))
continue;
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (fd == controller_fd) {
err(true, "Unexpected EOF from controller (fatal)");
} else {
err(false,
"Unexpected EOF from router with fd %d (shutting down)", fd);
finish_cmd();
exit(0);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
if (fd == controller_fd) {
/* Message from controller */
switch(code) {
case MSG_KILL:
chunk_free(msg);
#if RPT >= 1
report(1, "Received kill message from controller");
#endif
quit_agent(0, NULL);
break;
case MSG_DO_FLUSH:
chunk_free(msg);
#if RPT >= 2
report(2, "Received flush message from controller");
#endif
if (flush_helper) {
flush_helper(0, NULL);
}
break;
case MSG_GC_START:
/* Got notice that should initiate garbage collection.
Defer until current operation done */
#if RPT >= 3
report(3, "Deferring GC start");
#endif
chunk_free(msg);
gc_state = GC_DEFER;
break;
case MSG_GC_FINISH:
#if RPT >= 3
report(3,
"Unexpected GC_FINISH message from controller when waiting for result (ignored)");
#endif
chunk_free(msg);
break;
default:
chunk_free(msg);
err(false,
"Unknown message code %u from controller (ignored)", code);
}
} else {
dword_t dh;
word_t id = 0;
/* Must be message from router */
switch (code) {
case MSG_OPERATION:
chunk_free(msg);
err(false, "Received unexpected operation. Ignored.");
local_done = true;
break;
case MSG_OPERAND:
#if RPT >= 5
dh = chunk_get_dword(msg, 0);
id = msg_get_dheader_op_id(dh);
report(5, "Received operand with id 0x%lx", id);
#endif
rval = msg;
local_done = true;
break;
default:
chunk_free(msg);
err(false,
"Received message with unknown code %u (ignored)", code);
local_done = true;
}
}
}
}
return rval;
}
static void run_controller(char *infile_name) {
if (!start_cmd(infile_name))
return;
while (!cmd_done()) {
FD_ZERO(&set);
int fd;
word_t w;
unsigned ip;
unsigned port;
add_fd(listen_fd);
keyvalue_iterstart(new_conn_map);
/* Check for messages from newly connected clients, workers, and routers */
while (keyvalue_iternext(new_conn_map, &w, NULL)) {
fd = w;
add_fd(fd);
}
if (need_routers == 0) {
/* Accept messages from workers */
set_iterstart(worker_fd_set);
while (set_iternext(worker_fd_set, &w)) {
fd = w;
add_fd(fd);
}
/* Accept messages from clients */
set_iterstart(client_fd_set);
while (set_iternext(client_fd_set, &w)) {
fd = w;
add_fd(fd);
}
}
cmd_select(maxfd+1, &set, NULL, NULL, NULL);
for (fd = 0; fd <= maxfd; fd++) {
if (!FD_ISSET(fd, &set))
continue;
if (fd == listen_fd) {
unsigned ip;
int connfd = accept_connection(fd, &ip);
keyvalue_insert(new_conn_map, (word_t) connfd, (word_t) ip);
#if RPT >= 4
report(4, "Accepted new connection. Connfd = %d, IP = 0x%x",
connfd, ip);
#endif
continue;
}
bool eof;
chunk_ptr msg = chunk_read(fd, &eof);
if (eof) {
/* Unexpected EOF */
if (keyvalue_remove(new_conn_map, (word_t) fd, NULL, NULL)) {
err(false, "Unexpected EOF from new connection, fd %d", fd);
} else if (set_member(worker_fd_set, (word_t) fd, true)) {
err(false, "Unexpected EOF from connected worker, fd %d. Shutting down", fd);
/* Shut down system */
finish_cmd();
} else if (set_member(client_fd_set, (word_t) fd, true)) {
#if RPT >= 3
report(3, "Disconnection from client (fd %d)", fd);
#endif
if (need_client_fd_set && set_member(need_client_fd_set,
(word_t) fd, false)) {
#if RPT >= 3
report(3, "Removing client from GC activities");
#endif
handle_gc_msg(MSG_GC_FINISH, 0, fd, true);
}
} else {
err(false, "Unexpected EOF from unknown source, fd %d", fd);
}
close(fd);
continue;
}
if (msg == NULL) {
err(false, "Could not read chunk from fd %d (ignored)", fd);
continue;
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
#if RPT >= 5
report(5, "Received message with code %d from fd %d", code, fd);
#endif
if (keyvalue_remove(new_conn_map, (word_t) fd, NULL, &w)) {
ip = w;
chunk_free(msg);
/* Should be a registration message */
switch (code) {
case MSG_REGISTER_ROUTER:
if (need_routers == 0) {
err(false, "Unexpected router registration. (Ignored)");
close(fd);
break;
}
port = msg_get_header_port(h);
word_t node_id = msg_build_node_id(port, ip);
set_insert(router_addr_set, node_id);
set_insert(router_fd_set, (word_t) fd);
#if RPT >= 4
report(4, "Added router with fd %d. IP 0x%x. Port %u",
fd, ip, port);
#endif
need_routers --;
if (need_routers == 0) {
#if RPT >= 2
report(2, "All routers connected");
#endif
/* Have gotten all of the necessary routers.
Notify any registered workers */
set_iterstart(worker_fd_set);
int wfd;
while (set_iternext(worker_fd_set, &w)) {
wfd = w;
add_agent(wfd, false);
}
}
break;
case MSG_REGISTER_WORKER:
if (worker_fd_set->nelements >= worker_cnt) {
err(false, "Unexpected worker registration. (Ignored)");
close(fd);
break;
}
set_insert(worker_fd_set, (word_t) fd);
#if RPT >= 4
report(4, "Added worker with fd %d", fd);
#endif
if (need_routers == 0)
add_agent(fd, false);
break;
case MSG_REGISTER_CLIENT:
if (gc_state == GC_READY) {
set_insert(client_fd_set, (word_t) fd);
#if RPT >= 4
report(4, "Added client with fd %d", fd);
#endif
if (need_workers == 0)
add_agent(fd, true);
} else {
if (!defer_client_fd_set) {
defer_client_fd_set = word_set_new();
}
set_insert(defer_client_fd_set, (word_t) fd);
#if RPT >= 3
report(3, "Deferring client with fd %d until GC completed",
fd);
#endif
}
break;
default:
err(false, "Unexpected message code %u from new connection",
code);
break;
}
} else if (set_member(worker_fd_set, (word_t) fd, false)) {
/* Message from worker */
switch (code) {
unsigned agent;
unsigned gen;
case MSG_READY_WORKER:
chunk_free(msg);
if (need_workers == 0) {
err(false, "Unexpected worker ready. (Ignored)");
close(fd);
break;
}
need_workers--;
if (need_workers == 0) {
#if RPT >= 2
report(2, "All workers connected");
#endif
/* Notify any pending clients */
set_iterstart(client_fd_set);
int cfd;
while (set_iternext(client_fd_set, &w)) {
cfd = w;
add_agent(cfd, true);
}
}
break;
case MSG_STAT:
/* Message gets stashed away. Don't free it */
add_stat_message(msg);
break;
case MSG_CLIOP_ACK:
/* Worker acknowledging receipt of global operation info */
agent = msg_get_header_agent(h);
int client_fd = receive_global_op_worker_ack(agent);
if (client_fd >= 0) {
/* Have received complete set of acknowledgements. */
/* Send ack to client */
if (chunk_write(client_fd, msg)) {
#if RPT >= 6
report(6,
"Sent ack to client for global operation with id %u", agent);
#endif
} else {
err(false,
"Failed to send ack to client for global operation with id %u. Fd %d",
agent, client_fd);
}
}
chunk_free(msg);
break;
case MSG_GC_START:
case MSG_GC_FINISH:
handle_gc_msg(code, 0, fd, false);
chunk_free(msg);
break;
case MSG_GC_REQUEST:
gen = msg_get_header_generation(h);
chunk_free(msg);
handle_gc_msg(code, gen, fd, false);
break;
default:
chunk_free(msg);
err(false, "Unexpected message code %u from worker", code);
}
} else if (set_member(client_fd_set, (word_t) fd, false)) {
/* Message from client */
switch(code){
unsigned agent;
word_t w;
case MSG_KILL:
/* Shutdown entire system */
chunk_free(msg);
#if RPT >= 2
report(2, "Remote request to kill system");
#endif
finish_cmd();
return;
case MSG_DO_FLUSH:
/* Initiate a flush operation */
chunk_free(msg);
flush_requestor_fd = fd;
do_controller_flush_cmd(0, NULL);
break;
case MSG_CLIOP_DATA:
/* Request for global operation from client */
agent = msg_get_header_agent(h);
add_global_op(agent, fd);
/* Send message to all workers */
set_iterstart(worker_fd_set);
while (set_iternext(worker_fd_set, &w)) {
int worker_fd = (int) w;
if (chunk_write(worker_fd, msg)) {
#if RPT >= 6
report(6,
"Sent global operation information with id %u to worker with fd %d",
agent, worker_fd);
#endif
} else {
err(false,
"Failed to send global operation information with id %u to worker with fd %d",
agent, worker_fd);
}
}
chunk_free(msg);
break;
case MSG_CLIOP_ACK:
/* Completion of global operation by client */
agent = msg_get_header_agent(h);
/* Send message to all workers */
set_iterstart(worker_fd_set);
while (set_iternext(worker_fd_set, &w)) {
int worker_fd = (int) w;
if (chunk_write(worker_fd, msg)) {
#if RPT >= 6
report(6,
"Sent global operation acknowledgement with id %u to worker with fd %d",
agent, worker_fd);
#endif
} else {
err(false,
"Failed to send global operation acknowledgement with id %u to worker with fd %d",
agent, worker_fd);
}
}
chunk_free(msg);
break;
case MSG_GC_START:
case MSG_GC_FINISH:
handle_gc_msg(code, 0, fd, true);
chunk_free(msg);
break;
default:
err(false, "Unexpected message code %u from client", code);
}
} else {
chunk_free(msg);
err(false, "Unexpected message on fd %d (Ignored)", fd);
}
}
}
}
void* handle_client_request(void *arg)
{
struct msghdr *msg;
int soc = (int)arg;
char * data = (char *) malloc(MAX_BUF_SZ);
prepare_msg(0, &msg, data, MAX_BUF_SZ);
char * resp;
dfs_msg *dfsmsg;
int retval = recvmsg(soc, msg, 0);
if (retval == -1) {
printf("failed to receive message from client - errno-%d\n", errno);
} else {
dfsmsg = (dfs_msg*)msg->msg_iov[0].iov_base;
#ifdef DEBUG
printf("received message from client - %d bytes = %d\n", dfsmsg->msg_type, retval);
#endif
}
//extract the message type
print_msg(dfsmsg);
switch (dfsmsg->msg_type) {
case HEARTBEAT:
break;
case READ_DATA_REQ:
resp = (char*) malloc(sizeof(char)*MAX_BUF_SZ);
dfsmsg->status = chunk_read(msg->msg_iov[1].iov_base, resp);
msg->msg_iov[1].iov_base = resp;
msg->msg_iov[1].iov_len = strlen(resp)+1;
dfsmsg->msg_type = READ_DATA_RESP;
retval = sendmsg(soc, msg, 0);
if (retval == -1) {
printf("failed to send read reply to client - errno-%d\n", errno);
} else {
#ifdef DEBUG
printf("sent read reply to client\n");
#endif
}
break;
case WRITE_DATA_REQ:
dfsmsg->status = chunk_write(msg->msg_iov[1].iov_base);
dfsmsg->msg_type = WRITE_DATA_RESP;
retval = sendmsg(soc, msg, 0);
if (retval == -1) {
printf("failed to send write reply to client - errno-%d\n", errno);
} else {
#ifdef DEBUG
printf("sent write reply to client\n");
#endif
}
break;
case ROLLBACK_REQ:
dfsmsg->status = chunk_truncate(msg->msg_iov[1].iov_base);
dfsmsg->msg_type = ROLLBACK_RESP;
retval = sendmsg(soc, msg, 0);
if (retval == -1) {
printf("failed to send rollback reply to client - errno-%d\n", errno);
} else {
#ifdef DEBUG
printf("sent rollback reply to client\n");
#endif
}
break;
}
}
