static void accept_connection(void) { send_uri(); while (!connection) { int ret = 0; cci_event_t *event = NULL; ret = cci_get_event(endpoint, &event); if (ret == CCI_SUCCESS) { switch (event->type) { case CCI_EVENT_CONNECT_REQUEST: opts = *((options_t *) event->request.data_ptr); cci_accept(event, NULL); break; case CCI_EVENT_ACCEPT: assert(event->accept.status == CCI_SUCCESS); connection = event->accept.connection; ret = posix_memalign((void **)&buffer, 4096, opts.transfer_size); check_return(endpoint, "memalign buffer", ret, 1); memset(buffer, 'a', opts.transfer_size); ret = posix_memalign((void **)&ack, 4096, opts.ack_size); check_return(endpoint, "memalign buffer", ret, 1); memset(buffer, 'b', opts.ack_size); ret = cci_rma_register(endpoint, buffer, opts.transfer_size, CCI_FLAG_WRITE|CCI_FLAG_READ, &local); check_return(endpoint, "cci_rma_register", ret, 1); remote = &opts.rma_handle; break; default: fprintf(stderr, "%s: got %s event\n", __func__, cci_event_type_str(event->type)); } cci_return_event(event); } } return; }
static void poll_events(void) { int ret; cci_event_t *event; if (blocking) { FD_ZERO(&rfds); FD_SET(fd, &rfds); ret = select(nfds, &rfds, NULL, NULL, NULL); if (!ret) return; } ret = cci_get_event(endpoint, &event); if (ret == CCI_SUCCESS) { assert(event); switch (event->type) { case CCI_EVENT_SEND: if (event->send.status != CCI_SUCCESS) { fprintf(stderr, "RMA failed with %s.\n", cci_strerror(endpoint, event->send.status)); cci_disconnect(test); test = NULL; done = 1; } if (is_server) break; /* Client */ if (event->send.context == (void *)0xdeadbeef) { done = 1; break; } break; case CCI_EVENT_RECV: if (is_client) { hdr_t *h = (void*)event->recv.ptr; if (!ready) { ready = 1; memcpy((void*)&remote_rma_handle, &h->reply.handle, sizeof(remote_rma_handle)); } else { /* RMA status msg */ if (opts.method == RMA_WRITE) { if (h->status.crc != msg.check.crc) { fprintf(stderr, "Server reported " "CRC failed.\n" "Local CRC 0x%x != " "remote CRC 0x%x.\n" "count=%d current_size=%u\n", msg.check.crc, h->status.crc, count, current_size); } } else { uint32_t crc = 0; void *ptr = (void*)((uintptr_t)buffer + local_offset); /* Compute the CRC only on a valid buffer */ if (current_size + local_offset <= opts.reg_len) crc = crc32(0, ptr, current_size); else crc = 0; if (crc != h->status.crc) { fprintf(stderr, "Server reported " "CRC failed.\n" "Local CRC 0x%x != " "remote CRC 0x%x.\n" "count=%d current_size=%u\n", crc, h->status.crc, count, current_size); } } /* RMA completed */ count++; if (count < iters) { ret = cci_rma(test, &msg, msg_len, local_rma_handle, local_offset, &remote_rma_handle, remote_offset, current_size, NULL, opts.flags); check_return(endpoint, "cci_rma", ret, 1); } } } else { hdr_t *h = (void*)event->recv.ptr; /* is_server */ if (event->recv.len == 3) { done = 1; } else { uint32_t crc = 0; void *ptr = (void*)((uintptr_t)buffer + h->check.offset); /* RMA check request */ if ((h->check.len + h->check.offset) <= opts.reg_len) crc = crc32(0, ptr, h->check.len); else crc = 0; msg.status.type = MSG_RMA_STATUS; msg.status.crc = crc; if (opts.method == RMA_WRITE) { fprintf(stderr, "server: client crc=0x%x " "server crc=0x%x\n", h->check.crc, crc); } print_buffer(ptr, h->check.len); ret = cci_send(test, &msg, sizeof(msg.status), NULL, CCI_FLAG_SILENT); check_return(endpoint, "cci_send", ret, 1); } } break; case CCI_EVENT_CONNECT: if (event->connect.status != CCI_SUCCESS) { fprintf(stderr, "Connection rejected.\n"); exit(0); } if ((uintptr_t)event->connect.context == (uintptr_t)CONTROL) { control = event->connect.connection; } else { test = event->connect.connection; } if (control && test) connect_done = 1; break; case CCI_EVENT_CONNECT_REQUEST: fprintf(stderr, "Peer is reconnecting? Rejecting.\n"); cci_reject(event); break; default: fprintf(stderr, "ignoring event type %s\n", cci_event_type_str(event->type)); } cci_return_event(event); } return; }
static void do_server(void) { int ret = 0; hdr_t *h = NULL; while (!ready) { cci_event_t *event; if (blocking) { FD_ZERO(&rfds); FD_SET(fd, &rfds); ret = select(nfds, &rfds, NULL, NULL, NULL); if (!ret) return; } ret = cci_get_event(endpoint, &event); if (ret == CCI_SUCCESS) { switch (event->type) { case CCI_EVENT_CONNECT_REQUEST: { int which = 0; h = (void*)event->request.data_ptr; if (h->generic.type == MSG_CONN_REQ) { opts = h->request.opts; which = TEST; } else { which = CONTROL; } ret = cci_accept(event, (void*)((uintptr_t)which)); check_return(endpoint, "cci_accept", ret, 1); break; } case CCI_EVENT_ACCEPT: { if ((uintptr_t)event->accept.context == (uintptr_t)CONTROL) { control = event->accept.connection; } else { int len; test = event->accept.connection; len = opts.reg_len; ret = posix_memalign((void **)&buffer, 4096, len); check_return(endpoint, "memalign buffer", ret, 1); init_buffer(0); print_buffer(buffer, opts.reg_len); ret = cci_rma_register(endpoint, buffer, opts.reg_len, opts.method == RMA_WRITE ? CCI_FLAG_WRITE : CCI_FLAG_READ, &local_rma_handle); check_return(endpoint, "cci_rma_register", ret, 1); } if (test && control) { hdr_t msg; ready = 1; msg.reply.type = MSG_CONN_REPLY; msg.reply.handle = *local_rma_handle; ret = cci_send(test, &msg, sizeof(msg.reply), NULL, 0); check_return(endpoint, "cci_send", ret, 1); } break; } default: fprintf(stderr, "%s: ignoring unexpected event %s\n", __func__, cci_event_type_str(event->type)); break; } ret = cci_return_event(event); if (ret) fprintf(stderr, "cci_return_event() failed with %s\n", cci_strerror(endpoint, ret)); } } while (!done) poll_events(); ret = cci_rma_deregister(endpoint, local_rma_handle); check_return(endpoint, "cci_rma_deregister", ret, 1); printf("server done\n"); sleep(1); return; }
int main(int argc, char *argv[]) { int ret; uint32_t caps = 0; char *uri = NULL; cci_endpoint_t *endpoint = NULL; cci_os_handle_t ep_fd; cci_connection_t *connection = NULL; /* init */ ret = cci_init(CCI_ABI_VERSION, 0, &caps); if (ret) { fprintf(stderr, "cci_init() failed with %s\n", cci_strerror(NULL, ret)); exit(EXIT_FAILURE); } /* create an endpoint */ ret = cci_create_endpoint(NULL, 0, &endpoint, &ep_fd); if (ret) { fprintf(stderr, "cci_create_endpoint() failed with %s\n", cci_strerror(NULL, ret)); exit(EXIT_FAILURE); } ret = cci_get_opt(endpoint, CCI_OPT_ENDPT_URI, &uri); if (ret) { fprintf(stderr, "cci_get_opt() failed with %s\n", cci_strerror(endpoint, ret)); exit(EXIT_FAILURE); } printf("Opened %s\n", uri); while (1) { cci_event_t *event; ret = cci_get_event(endpoint, &event); if (ret != CCI_SUCCESS) { if (ret != CCI_EAGAIN) { fprintf(stderr, "cci_get_event() returned %s", cci_strerror(endpoint, ret)); } continue; } /* fprintf(stderr, "ret: %d, event: %d (RECV:%d, SEND:%d, CONNECT_REQ:%d, ACCEPT:%d)\n", */ /* ret, event->type, CCI_EVENT_RECV, CCI_EVENT_SEND, CCI_EVENT_CONNECT_REQUEST, */ /* CCI_EVENT_ACCEPT); */ fprintf(stderr, "Event: %s, ret:%d len: %lu\n", cci_event_type_str(event->type), ret, event->recv.len); switch (event->type) { case CCI_EVENT_RECV: { // fprintf(stderr, "=====: %p %p %d\n", buffer, event->recv.ptr, event->recv.len); // memcpy(buffer, // event->recv.ptr, event->recv.len); // fprintf(stderr, "=====\n"); // buffer[event->recv.len] = 0; // printf("recv'd \n"); /* echo the message to the client */ ret = cci_send(connection, event->recv.ptr, event->recv.len, NULL, 0); if (ret != CCI_SUCCESS) fprintf(stderr, "send returned %s\n", cci_strerror(endpoint, ret)); break; } case CCI_EVENT_SEND: // printf("completed send\n"); break; case CCI_EVENT_CONNECT_REQUEST: { int accept = 1; if (accept) { ret = cci_accept(event, NULL); if (ret != CCI_SUCCESS) { fprintf(stderr, "cci_accept() returned %s", cci_strerror(endpoint, ret)); } } else { cci_reject(event); } } break; case CCI_EVENT_ACCEPT: connection = event->accept.connection; if (!buffer) { buffer = calloc(1, connection->max_send_size + 1); fprintf(stderr, "allocated buffer: %p , size %d", buffer, connection->max_send_size + 1); /* check for buffer ... */ } else { fprintf(stderr, "Not allocated\n"); } break; default: fprintf(stderr, "unexpected event %d", event->type); break; } cci_return_event(event); } /* clean up */ cci_destroy_endpoint(endpoint); /* add cci_finalize() here */ return 0; }