chunk_ptr chunk_read_unbuffered(int fd, bool *eofp) { bufferReadBool = 0; chunk_ptr p = chunk_read(fd, eofp); bufferReadBool = 1; return p; }
void run_worker() { while (true) { /* Select among controller port, and connections to routers */ FD_ZERO(&cset); maxcfd = 0; add_cfd(controller_fd); unsigned ridx; for (ridx = 0; ridx < nrouters; ridx++) add_cfd(router_fd_array[ridx]); buf_select(maxcfd+1, &cset, NULL, NULL, NULL); int fd; for (fd = 0; fd <= maxcfd; fd++) { if (!FD_ISSET(fd, &cset)) continue; bool eof; chunk_ptr msg = chunk_read(fd, &eof); if (eof) { /* Unexpected EOF */ if (fd == controller_fd) { err(true, "Unexpected EOF from controller (fatal)"); } else { err(false, "Unexpected EOF from router with fd %d (shutting down)", fd); finish_cmd(); exit(0); } close(fd); continue; } if (msg == NULL) { err(false, "Could not read chunk from fd %d (ignored)", fd); continue; } word_t h = chunk_get_word(msg, 0); /* Rely on fact that following message fields in same location for both single and double-word headers */ unsigned code = msg_get_header_code(h); unsigned agent = msg_get_header_agent(h); unsigned opcode = msg_get_header_opcode(h); if (fd == controller_fd) { /* Message from controller */ switch(code) { case MSG_KILL: chunk_free(msg); #if RPT >= 1 report(1, "Received kill message from controller"); #endif quit_agent(0, NULL); return; case MSG_DO_FLUSH: chunk_free(msg); #if RPT >= 5 report(5, "Received flush message from controller"); #endif if (flush_helper) { chunk_ptr msg = flush_helper(); if (!msg) break; if (chunk_write(controller_fd, msg)) { #if RPT >= 5 report(5, "Sent statistics information to controller"); #endif } else { err(false, "Failed to send statistics information to controller"); } chunk_free(msg); } break; case MSG_CLIOP_DATA: report(5, "Received client operation data. Agent = %u", agent); word_t *data = &msg->words[1]; unsigned nword = msg->length - 1; if (gop_start_helper) gop_start_helper(agent, opcode, nword, data); chunk_free(msg); chunk_ptr rmsg = msg_new_cliop_ack(agent); if (chunk_write(controller_fd, rmsg)) { #if RPT >= 5 report(5, "Acknowledged client operation data. Agent = %u", agent); #endif } else { err(false, "Failed to acknowledge client operation data. Agent = %u", agent); } chunk_free(rmsg); break; case MSG_CLIOP_ACK: #if RPT >= 5 report(5, "Received client operation ack. Agent = %u", agent); #endif if (gop_finish_helper) gop_finish_helper(agent); chunk_free(msg); break; case MSG_GC_START: chunk_free(msg); gc_start(code); break; case MSG_GC_FINISH: chunk_free(msg); gc_finish(code); break; default: chunk_free(msg); err(false, "Unknown message code %u from controller (ignored)", code); } } else { /* Must be message from router */ switch (code) { case MSG_OPERATION: receive_operation(msg); break; case MSG_OPERAND: receive_operand(msg); break; default: chunk_free(msg); err(false, "Received message with unknown code %u (ignored)", code); } } } } quit_agent(0, NULL); }
/* Client command loop only considers console inputs + controller messages */ void run_client(char *infile_name) { if (!start_cmd(infile_name)) return; while (!cmd_done()) { /* Select among controller port, and connections to routers */ FD_ZERO(&cset); maxcfd = 0; add_cfd(controller_fd); cmd_select(maxcfd+1, &cset, NULL, NULL, NULL); if (cmd_done()) break; int fd; for (fd = 0; fd <= maxcfd; fd++) { if (!FD_ISSET(fd, &cset)) continue; bool eof; chunk_ptr msg = chunk_read(fd, &eof); if (eof) { /* Unexpected EOF */ if (fd == controller_fd) { err(true, "Unexpected EOF from controller (fatal)"); } else { err(false, "Unexpected EOF from unexpected source. fd %d (ignored)", fd); } close(fd); continue; } if (msg == NULL) { err(false, "Could not read chunk from fd %d (ignored)", fd); continue; } word_t h = chunk_get_word(msg, 0); unsigned code = msg_get_header_code(h); if (fd == controller_fd) { /* Message from controller */ switch(code) { case MSG_DO_FLUSH: chunk_free(msg); #if RPT >= 5 report(5, "Received flush message from controller"); #endif if (flush_helper) { flush_helper(); } break; case MSG_STAT: #if RPT >= 5 report(5, "Received summary statistics from controller"); #endif if (stat_helper) { /* Get a copy of the byte usage from mem. allocator */ stat_helper(msg); } chunk_free(msg); /* Client can proceed with next command */ unblock_console(); break; case MSG_KILL: chunk_free(msg); #if RPT >= 1 report(1, "Received kill message from controller"); #endif finish_cmd(); break; case MSG_GC_START: chunk_free(msg); gc_start(code); break; case MSG_GC_FINISH: chunk_free(msg); gc_finish(code); break; default: chunk_free(msg); err(false, "Unknown message code %u from controller (ignored)", code); } } else { err(false, "Received message from unknown source. fd %d (ignored)", fd); close(fd); continue; } } } }
/* Fire an operation and wait for returned operand */ chunk_ptr fire_and_wait_defer(chunk_ptr msg) { chunk_ptr rval = NULL; if (!send_op(msg)) { err(false, "Failed to send message"); return NULL; } bool local_done = false; while (!(local_done || cmd_done())) { /* Select among controller port, and connections to routers. Do not accept console input */ FD_ZERO(&rset); maxrfd = 0; add_rfd(controller_fd); unsigned ridx; for (ridx = 0; ridx < nrouters; ridx++) add_rfd(router_fd_array[ridx]); buf_select(maxrfd+1, &rset, NULL, NULL, NULL); int fd; for (fd = 0; fd <= maxrfd; fd++) { if (!FD_ISSET(fd, &rset)) continue; bool eof; chunk_ptr msg = chunk_read(fd, &eof); if (eof) { /* Unexpected EOF */ if (fd == controller_fd) { err(true, "Unexpected EOF from controller (fatal)"); } else { err(false, "Unexpected EOF from router with fd %d (shutting down)", fd); finish_cmd(); exit(0); } close(fd); continue; } if (msg == NULL) { err(false, "Could not read chunk from fd %d (ignored)", fd); continue; } word_t h = chunk_get_word(msg, 0); unsigned code = msg_get_header_code(h); if (fd == controller_fd) { /* Message from controller */ switch(code) { case MSG_KILL: chunk_free(msg); #if RPT >= 1 report(1, "Received kill message from controller"); #endif quit_agent(0, NULL); break; case MSG_DO_FLUSH: chunk_free(msg); #if RPT >= 2 report(2, "Received flush message from controller"); #endif if (flush_helper) { flush_helper(0, NULL); } break; case MSG_GC_START: /* Got notice that should initiate garbage collection. Defer until current operation done */ #if RPT >= 3 report(3, "Deferring GC start"); #endif chunk_free(msg); gc_state = GC_DEFER; break; case MSG_GC_FINISH: #if RPT >= 3 report(3, "Unexpected GC_FINISH message from controller when waiting for result (ignored)"); #endif chunk_free(msg); break; default: chunk_free(msg); err(false, "Unknown message code %u from controller (ignored)", code); } } else { dword_t dh; word_t id = 0; /* Must be message from router */ switch (code) { case MSG_OPERATION: chunk_free(msg); err(false, "Received unexpected operation. Ignored."); local_done = true; break; case MSG_OPERAND: #if RPT >= 5 dh = chunk_get_dword(msg, 0); id = msg_get_dheader_op_id(dh); report(5, "Received operand with id 0x%lx", id); #endif rval = msg; local_done = true; break; default: chunk_free(msg); err(false, "Received message with unknown code %u (ignored)", code); local_done = true; } } } } return rval; }
static void run_controller(char *infile_name) { if (!start_cmd(infile_name)) return; while (!cmd_done()) { FD_ZERO(&set); int fd; word_t w; unsigned ip; unsigned port; add_fd(listen_fd); keyvalue_iterstart(new_conn_map); /* Check for messages from newly connected clients, workers, and routers */ while (keyvalue_iternext(new_conn_map, &w, NULL)) { fd = w; add_fd(fd); } if (need_routers == 0) { /* Accept messages from workers */ set_iterstart(worker_fd_set); while (set_iternext(worker_fd_set, &w)) { fd = w; add_fd(fd); } /* Accept messages from clients */ set_iterstart(client_fd_set); while (set_iternext(client_fd_set, &w)) { fd = w; add_fd(fd); } } cmd_select(maxfd+1, &set, NULL, NULL, NULL); for (fd = 0; fd <= maxfd; fd++) { if (!FD_ISSET(fd, &set)) continue; if (fd == listen_fd) { unsigned ip; int connfd = accept_connection(fd, &ip); keyvalue_insert(new_conn_map, (word_t) connfd, (word_t) ip); #if RPT >= 4 report(4, "Accepted new connection. Connfd = %d, IP = 0x%x", connfd, ip); #endif continue; } bool eof; chunk_ptr msg = chunk_read(fd, &eof); if (eof) { /* Unexpected EOF */ if (keyvalue_remove(new_conn_map, (word_t) fd, NULL, NULL)) { err(false, "Unexpected EOF from new connection, fd %d", fd); } else if (set_member(worker_fd_set, (word_t) fd, true)) { err(false, "Unexpected EOF from connected worker, fd %d. Shutting down", fd); /* Shut down system */ finish_cmd(); } else if (set_member(client_fd_set, (word_t) fd, true)) { #if RPT >= 3 report(3, "Disconnection from client (fd %d)", fd); #endif if (need_client_fd_set && set_member(need_client_fd_set, (word_t) fd, false)) { #if RPT >= 3 report(3, "Removing client from GC activities"); #endif handle_gc_msg(MSG_GC_FINISH, 0, fd, true); } } else { err(false, "Unexpected EOF from unknown source, fd %d", fd); } close(fd); continue; } if (msg == NULL) { err(false, "Could not read chunk from fd %d (ignored)", fd); continue; } word_t h = chunk_get_word(msg, 0); unsigned code = msg_get_header_code(h); #if RPT >= 5 report(5, "Received message with code %d from fd %d", code, fd); #endif if (keyvalue_remove(new_conn_map, (word_t) fd, NULL, &w)) { ip = w; chunk_free(msg); /* Should be a registration message */ switch (code) { case MSG_REGISTER_ROUTER: if (need_routers == 0) { err(false, "Unexpected router registration. (Ignored)"); close(fd); break; } port = msg_get_header_port(h); word_t node_id = msg_build_node_id(port, ip); set_insert(router_addr_set, node_id); set_insert(router_fd_set, (word_t) fd); #if RPT >= 4 report(4, "Added router with fd %d. IP 0x%x. Port %u", fd, ip, port); #endif need_routers --; if (need_routers == 0) { #if RPT >= 2 report(2, "All routers connected"); #endif /* Have gotten all of the necessary routers. Notify any registered workers */ set_iterstart(worker_fd_set); int wfd; while (set_iternext(worker_fd_set, &w)) { wfd = w; add_agent(wfd, false); } } break; case MSG_REGISTER_WORKER: if (worker_fd_set->nelements >= worker_cnt) { err(false, "Unexpected worker registration. (Ignored)"); close(fd); break; } set_insert(worker_fd_set, (word_t) fd); #if RPT >= 4 report(4, "Added worker with fd %d", fd); #endif if (need_routers == 0) add_agent(fd, false); break; case MSG_REGISTER_CLIENT: if (gc_state == GC_READY) { set_insert(client_fd_set, (word_t) fd); #if RPT >= 4 report(4, "Added client with fd %d", fd); #endif if (need_workers == 0) add_agent(fd, true); } else { if (!defer_client_fd_set) { defer_client_fd_set = word_set_new(); } set_insert(defer_client_fd_set, (word_t) fd); #if RPT >= 3 report(3, "Deferring client with fd %d until GC completed", fd); #endif } break; default: err(false, "Unexpected message code %u from new connection", code); break; } } else if (set_member(worker_fd_set, (word_t) fd, false)) { /* Message from worker */ switch (code) { unsigned agent; unsigned gen; case MSG_READY_WORKER: chunk_free(msg); if (need_workers == 0) { err(false, "Unexpected worker ready. (Ignored)"); close(fd); break; } need_workers--; if (need_workers == 0) { #if RPT >= 2 report(2, "All workers connected"); #endif /* Notify any pending clients */ set_iterstart(client_fd_set); int cfd; while (set_iternext(client_fd_set, &w)) { cfd = w; add_agent(cfd, true); } } break; case MSG_STAT: /* Message gets stashed away. Don't free it */ add_stat_message(msg); break; case MSG_CLIOP_ACK: /* Worker acknowledging receipt of global operation info */ agent = msg_get_header_agent(h); int client_fd = receive_global_op_worker_ack(agent); if (client_fd >= 0) { /* Have received complete set of acknowledgements. */ /* Send ack to client */ if (chunk_write(client_fd, msg)) { #if RPT >= 6 report(6, "Sent ack to client for global operation with id %u", agent); #endif } else { err(false, "Failed to send ack to client for global operation with id %u. Fd %d", agent, client_fd); } } chunk_free(msg); break; case MSG_GC_START: case MSG_GC_FINISH: handle_gc_msg(code, 0, fd, false); chunk_free(msg); break; case MSG_GC_REQUEST: gen = msg_get_header_generation(h); chunk_free(msg); handle_gc_msg(code, gen, fd, false); break; default: chunk_free(msg); err(false, "Unexpected message code %u from worker", code); } } else if (set_member(client_fd_set, (word_t) fd, false)) { /* Message from client */ switch(code){ unsigned agent; word_t w; case MSG_KILL: /* Shutdown entire system */ chunk_free(msg); #if RPT >= 2 report(2, "Remote request to kill system"); #endif finish_cmd(); return; case MSG_DO_FLUSH: /* Initiate a flush operation */ chunk_free(msg); flush_requestor_fd = fd; do_controller_flush_cmd(0, NULL); break; case MSG_CLIOP_DATA: /* Request for global operation from client */ agent = msg_get_header_agent(h); add_global_op(agent, fd); /* Send message to all workers */ set_iterstart(worker_fd_set); while (set_iternext(worker_fd_set, &w)) { int worker_fd = (int) w; if (chunk_write(worker_fd, msg)) { #if RPT >= 6 report(6, "Sent global operation information with id %u to worker with fd %d", agent, worker_fd); #endif } else { err(false, "Failed to send global operation information with id %u to worker with fd %d", agent, worker_fd); } } chunk_free(msg); break; case MSG_CLIOP_ACK: /* Completion of global operation by client */ agent = msg_get_header_agent(h); /* Send message to all workers */ set_iterstart(worker_fd_set); while (set_iternext(worker_fd_set, &w)) { int worker_fd = (int) w; if (chunk_write(worker_fd, msg)) { #if RPT >= 6 report(6, "Sent global operation acknowledgement with id %u to worker with fd %d", agent, worker_fd); #endif } else { err(false, "Failed to send global operation acknowledgement with id %u to worker with fd %d", agent, worker_fd); } } chunk_free(msg); break; case MSG_GC_START: case MSG_GC_FINISH: handle_gc_msg(code, 0, fd, true); chunk_free(msg); break; default: err(false, "Unexpected message code %u from client", code); } } else { chunk_free(msg); err(false, "Unexpected message on fd %d (Ignored)", fd); } } } }
void* handle_client_request(void *arg) { struct msghdr *msg; int soc = (int)arg; char * data = (char *) malloc(MAX_BUF_SZ); prepare_msg(0, &msg, data, MAX_BUF_SZ); char * resp; dfs_msg *dfsmsg; int retval = recvmsg(soc, msg, 0); if (retval == -1) { printf("failed to receive message from client - errno-%d\n", errno); } else { dfsmsg = (dfs_msg*)msg->msg_iov[0].iov_base; #ifdef DEBUG printf("received message from client - %d bytes = %d\n", dfsmsg->msg_type, retval); #endif } //extract the message type print_msg(dfsmsg); switch (dfsmsg->msg_type) { case HEARTBEAT: break; case READ_DATA_REQ: resp = (char*) malloc(sizeof(char)*MAX_BUF_SZ); dfsmsg->status = chunk_read(msg->msg_iov[1].iov_base, resp); msg->msg_iov[1].iov_base = resp; msg->msg_iov[1].iov_len = strlen(resp)+1; dfsmsg->msg_type = READ_DATA_RESP; retval = sendmsg(soc, msg, 0); if (retval == -1) { printf("failed to send read reply to client - errno-%d\n", errno); } else { #ifdef DEBUG printf("sent read reply to client\n"); #endif } break; case WRITE_DATA_REQ: dfsmsg->status = chunk_write(msg->msg_iov[1].iov_base); dfsmsg->msg_type = WRITE_DATA_RESP; retval = sendmsg(soc, msg, 0); if (retval == -1) { printf("failed to send write reply to client - errno-%d\n", errno); } else { #ifdef DEBUG printf("sent write reply to client\n"); #endif } break; case ROLLBACK_REQ: dfsmsg->status = chunk_truncate(msg->msg_iov[1].iov_base); dfsmsg->msg_type = ROLLBACK_RESP; retval = sendmsg(soc, msg, 0); if (retval == -1) { printf("failed to send rollback reply to client - errno-%d\n", errno); } else { #ifdef DEBUG printf("sent rollback reply to client\n"); #endif } break; } }