int main(int argc, char **argv) { struct sockaddr_in addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *listener = NULL; struct rdma_event_channel *ec = NULL; uint16_t port = 0; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_bind_addr(listener, (struct sockaddr *)&addr)); TEST_NZ(rdma_listen(listener, 10)); /* backlog=10 is arbitrary */ port = ntohs(rdma_get_src_port(listener)); printf("listening on port %d.\n", port); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) break; } rdma_destroy_id(listener); rdma_destroy_event_channel(ec); return 0; }
/* create a portal, after listening starts all events * are received in isert_cm_evt_handler() */ struct isert_portal *isert_portal_create(void) { struct isert_portal *portal; struct rdma_cm_id *cm_id; int err; if (unlikely(!try_module_get(THIS_MODULE))) { pr_err("Unable increment module reference\n"); portal = ERR_PTR(-EINVAL); goto out; } portal = kzalloc(sizeof(*portal), GFP_KERNEL); if (unlikely(!portal)) { pr_err("Unable to allocate struct portal\n"); portal = ERR_PTR(-ENOMEM); goto err_alloc; } #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) && \ (!defined(RHEL_MAJOR) || RHEL_MAJOR -0 <= 5) cm_id = rdma_create_id(isert_cm_evt_handler, portal, RDMA_PS_TCP); #else cm_id = rdma_create_id(isert_cm_evt_handler, portal, RDMA_PS_TCP, IB_QPT_RC); #endif if (unlikely(IS_ERR(cm_id))) { err = PTR_ERR(cm_id); pr_err("Failed to create rdma id, err:%d\n", err); goto create_id_err; } portal->cm_id = cm_id; INIT_LIST_HEAD(&portal->conn_list); isert_portal_list_add(portal); #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0) rdma_set_afonly(cm_id, 1); #endif pr_info("Created iser portal cm_id:%p\n", cm_id); out: return portal; create_id_err: kfree(portal); portal = ERR_PTR(err); err_alloc: module_put(THIS_MODULE); goto out; }
int ibrdma_transfer(struct transfer_info *tfi, int num_tfi) { struct addrinfo *addr; struct rdma_cm_id *cmid= NULL; struct rdma_event_channel *ec = NULL; struct rdma_conn_param cm_params; TEST_NZ(getaddrinfo(host, port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED)); freeaddrinfo(addr); build_connection(cmid); TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED)); build_params(&cm_params); TEST_NZ(rdma_connect(cmid, &cm_params)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED)); on_connect(cmid->context); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED)); rdma_destroy_id(&cmid); rdma_destroy_event_channel(&ec); return 0; }
int xfer_rdma_init(struct xfer_data *data) { int duplex = 0; /* Get the PID and prepend it to every output on stdout/stderr * This helps to parse output when multiple client/server are * run from single host */ pid = getpid(); printf("%d: | port=%d | ib_port=%d | tx_depth=%d | sl=%d | duplex=%d | cma=%d |\n", pid, data->port, data->ib_port, data->tx_depth, sl, duplex, data->use_cma); srand48(pid * time(NULL)); page_size = sysconf(_SC_PAGESIZE); if (data->use_cma) { data->cm_channel = rdma_create_event_channel(); if (!data->cm_channel) { fprintf(stderr, "%d:%s: rdma_create_event_channel failed\n", pid, __func__); return -1; } if (rdma_create_id(data->cm_channel, &data->cm_id, NULL, RDMA_PS_TCP)) { fprintf(stderr, "%d:%s: rdma_create_id failed\n", pid, __func__); return -1; } } else { // use an alternative to CMA here } return 0; }
//static int run(int argc, char **argv) int ibrdma_send(char* host, char* port, void* data, uint64_t size) { struct addrinfo *addr; struct rdma_cm_id *cmid= NULL; struct rdma_event_channel *ec = NULL; struct rdma_conn_param cm_params; TEST_NZ(getaddrinfo(host, port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED)); freeaddrinfo(addr); build_connection(cmid); TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED)); build_params(&cm_params); TEST_NZ(rdma_connect(cmid, &cm_params)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED)); on_connect(cmid->context); /* Init MSG send to start RDMA*/ init_tfile(data, size); send_init(cmid->context); /*----------------------------*/ TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED)); rdma_destroy_id(cmid); rdma_destroy_event_channel(ec); return 0; }
//static int run(int argc, char **argv) //int RDMA_Connect(struct RDMA_communicator *comm, struct RDMA_param *param) int RDMA_Active_Init(struct RDMA_communicator *comm, struct RDMA_param *param) { struct addrinfo *addr; // struct rdma_cm_id *cm_id= NULL; // struct rdma_event_channel *ec = NULL; struct rdma_conn_param cm_params; char port[8]; // int i,j; sprintf(port, "%d", RDMA_PORT); TEST_NZ(getaddrinfo(param->host, port, NULL, &addr)); TEST_Z(comm->ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(comm->ec, &(comm->cm_id), NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(comm->cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ADDR_RESOLVED)); freeaddrinfo(addr); build_connection(comm->cm_id); TEST_NZ(rdma_resolve_route(comm->cm_id, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ROUTE_RESOLVED)); build_params(&cm_params); TEST_NZ(rdma_connect(comm->cm_id, &cm_params)); TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ESTABLISHED)); // on_connect(cm_id->context); return 0; }
int main(int argc, char **argv) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; if (argc != 3) die("usage: client <server-address> <server-port>"); TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) break; } rdma_destroy_event_channel(ec); return 0; }
/** * starts the process of connecting to the target * sleeps untill the connection is established or rejected */ int iser_connect(struct iser_conn *ib_conn, struct sockaddr_in *src_addr, struct sockaddr_in *dst_addr, int non_blocking) { struct sockaddr *src, *dst; int err = 0; sprintf(ib_conn->name, "%pI4:%d", &dst_addr->sin_addr.s_addr, dst_addr->sin_port); /* the device is known only --after-- address resolution */ ib_conn->device = NULL; iser_err("connecting to: %pI4, port 0x%x\n", &dst_addr->sin_addr, dst_addr->sin_port); ib_conn->state = ISER_CONN_PENDING; iser_conn_get(ib_conn); /* ref ib conn's cma id */ ib_conn->cma_id = rdma_create_id(iser_cma_handler, (void *)ib_conn, RDMA_PS_TCP); if (IS_ERR(ib_conn->cma_id)) { err = PTR_ERR(ib_conn->cma_id); iser_err("rdma_create_id failed: %d\n", err); goto id_failure; } src = (struct sockaddr *)src_addr; dst = (struct sockaddr *)dst_addr; err = rdma_resolve_addr(ib_conn->cma_id, src, dst, 1000); if (err) { iser_err("rdma_resolve_addr failed: %d\n", err); goto addr_failure; } if (!non_blocking) { wait_event_interruptible(ib_conn->wait, (ib_conn->state != ISER_CONN_PENDING)); if (ib_conn->state != ISER_CONN_UP) { err = -EIO; goto connect_failure; } } mutex_lock(&ig.connlist_mutex); list_add(&ib_conn->conn_list, &ig.connlist); mutex_unlock(&ig.connlist_mutex); return 0; id_failure: ib_conn->cma_id = NULL; addr_failure: ib_conn->state = ISER_CONN_DOWN; connect_failure: iser_conn_release(ib_conn, 1); return err; }
static int alloc_nodes(void) { int ret, i; test.nodes = malloc(sizeof *test.nodes * connections); if (!test.nodes) { printf("cmatose: unable to allocate memory for test nodes\n"); return -ENOMEM; } memset(test.nodes, 0, sizeof *test.nodes * connections); for (i = 0; i < connections; i++) { test.nodes[i].id = i; if (dst_addr) { ret = rdma_create_id(test.channel, &test.nodes[i].cma_id, &test.nodes[i], hints.ai_port_space); if (ret) goto err; } } return 0; err: while (--i >= 0) rdma_destroy_id(test.nodes[i].cma_id); free(test.nodes); return ret; }
static int fi_ibv_pep_setname(fid_t pep_fid, void *addr, size_t addrlen) { struct fi_ibv_pep *pep; int ret; pep = container_of(pep_fid, struct fi_ibv_pep, pep_fid); if (pep->src_addrlen && (addrlen != pep->src_addrlen)) { FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "addrlen expected: %d, got: %d.\n", pep->src_addrlen, addrlen); return -FI_EINVAL; } /* Re-create id if already bound */ if (pep->bound) { ret = rdma_destroy_id(pep->id); if (ret) { FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to destroy previous rdma_cm_id\n"); return -errno; } ret = rdma_create_id(NULL, &pep->id, NULL, RDMA_PS_TCP); if (ret) { FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to create rdma_cm_id\n"); return -errno; } } ret = rdma_bind_addr(pep->id, (struct sockaddr *)addr); if (ret) { FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to bind addres to rdma_cm_id\n"); return -errno; } return 0; }
int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn_userdata) { struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv); struct ibw_conn_priv *pconn = NULL; int rc; assert(conn!=NULL); conn->conn_userdata = conn_userdata; pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); DEBUG(DEBUG_DEBUG, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr), ntohs(serv_addr->sin_port))); /* clean previous - probably half - initialization */ if (ibw_conn_priv_destruct(pconn)) { DEBUG(DEBUG_ERR, ("ibw_connect/ibw_pconn_destruct failed for cm_id=%p\n", pconn->cm_id)); return -1; } /* init cm */ #if RDMA_USER_CM_MAX_ABI_VERSION >= 2 rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP); #else rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn); #endif if (rc) { rc = errno; sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc); talloc_free(conn); return -1; } DEBUG(DEBUG_DEBUG, ("ibw_connect: rdma_create_id succeeded, cm_id=%p\n", pconn->cm_id)); rc = rdma_resolve_addr(pconn->cm_id, NULL, (struct sockaddr *) serv_addr, 2000); if (rc) { sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc); DEBUG(DEBUG_ERR, (ibw_lasterr)); talloc_free(conn); return -1; } /* continued at RDMA_CM_EVENT_ADDR_RESOLVED */ return 0; }
/***************************************************************************//** * Connection server * ******************************************************************************/ static struct rdma_conn * build_connection(struct thread_context *ctx) { struct rdma_conn *c = calloc(1, sizeof(struct rdma_conn)); if (0 != rdma_create_id(NULL, &c->id, c, RDMA_PS_TCP)) { perror("rdma_create_id()"); return NULL; } struct rdma_addrinfo hints = { .ai_port_space = RDMA_PS_TCP },
/* * Create a listening RDMA service endpoint. */ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, struct net *net, struct sockaddr *sa, int salen, int flags) { struct rdma_cm_id *listen_id; struct svcxprt_rdma *cma_xprt; struct svc_xprt *xprt; int ret; dprintk("svcrdma: Creating RDMA socket\n"); if (sa->sa_family != AF_INET) { dprintk("svcrdma: Address family %d is not supported.\n", sa->sa_family); return ERR_PTR(-EAFNOSUPPORT); } cma_xprt = rdma_create_xprt(serv, 1); if (!cma_xprt) return ERR_PTR(-ENOMEM); xprt = &cma_xprt->sc_xprt; listen_id = rdma_create_id(rdma_listen_handler, cma_xprt, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(listen_id)) { ret = PTR_ERR(listen_id); dprintk("svcrdma: rdma_create_id failed = %d\n", ret); goto err0; } ret = rdma_bind_addr(listen_id, sa); if (ret) { dprintk("svcrdma: rdma_bind_addr failed = %d\n", ret); goto err1; } cma_xprt->sc_cm_id = listen_id; ret = rdma_listen(listen_id, RPCRDMA_LISTEN_BACKLOG); if (ret) { dprintk("svcrdma: rdma_listen failed = %d\n", ret); goto err1; } /* * We need to use the address from the cm_id in case the * caller specified 0 for the port number. */ sa = (struct sockaddr *)&cma_xprt->sc_cm_id->route.addr.src_addr; svc_xprt_set_local(&cma_xprt->sc_xprt, sa, salen); return &cma_xprt->sc_xprt; err1: rdma_destroy_id(listen_id); err0: kfree(cma_xprt); return ERR_PTR(ret); }
static int fi_ibv_rdm_cm_init(struct fi_ibv_rdm_cm* cm, const struct rdma_addrinfo* rai) { struct sockaddr_in* src_addr = (struct sockaddr_in*)rai->ai_src_addr; cm->ec = rdma_create_event_channel(); if (!cm->ec) { VERBS_INFO(FI_LOG_EP_CTRL, "Failed to create listener event channel: %s\n", strerror(errno)); return -FI_EOTHER; } if (fi_fd_nonblock(cm->ec->fd) != 0) { VERBS_INFO_ERRNO(FI_LOG_EP_CTRL, "fcntl", errno); return -FI_EOTHER; } if (rdma_create_id(cm->ec, &cm->listener, NULL, RDMA_PS_TCP)) { VERBS_INFO(FI_LOG_EP_CTRL, "Failed to create cm listener: %s\n", strerror(errno)); return -FI_EOTHER; } if (fi_ibv_rdm_find_ipoib_addr(src_addr, cm)) { VERBS_INFO(FI_LOG_EP_CTRL, "Failed to find correct IPoIB address\n"); return -FI_ENODEV; } cm->my_addr.sin_port = src_addr->sin_port; char my_ipoib_addr_str[INET6_ADDRSTRLEN]; inet_ntop(cm->my_addr.sin_family, &cm->my_addr.sin_addr.s_addr, my_ipoib_addr_str, INET_ADDRSTRLEN); VERBS_INFO(FI_LOG_EP_CTRL, "My IPoIB: %s\n", my_ipoib_addr_str); if (rdma_bind_addr(cm->listener, (struct sockaddr *)&cm->my_addr)) { VERBS_INFO(FI_LOG_EP_CTRL, "Failed to bind cm listener to my IPoIB addr %s: %s\n", my_ipoib_addr_str, strerror(errno)); return -FI_EOTHER; } if (!cm->my_addr.sin_port) { cm->my_addr.sin_port = rdma_get_src_port(cm->listener); } assert(cm->my_addr.sin_family == AF_INET); VERBS_INFO(FI_LOG_EP_CTRL, "My ep_addr: %s:%u\n", inet_ntoa(cm->my_addr.sin_addr), ntohs(cm->my_addr.sin_port)); return FI_SUCCESS; }
static int run_server(void) { struct rdma_cm_id *listen_id; int i, ret; printf("udaddy: starting server\n"); ret = rdma_create_id(test.channel, &listen_id, &test, port_space); if (ret) { perror("udaddy: listen request failed"); return ret; } if (src_addr) { ret = get_addr(src_addr, &test.src_in); if (ret) goto out; } else test.src_in.sin_family = PF_INET; test.src_in.sin_port = port; ret = rdma_bind_addr(listen_id, test.src_addr); if (ret) { perror("udaddy: bind address failed"); return ret; } ret = rdma_listen(listen_id, 0); if (ret) { perror("udaddy: failure trying to listen"); goto out; } connect_events(); if (message_count) { printf("receiving data transfers\n"); ret = poll_cqs(); if (ret) goto out; printf("sending replies\n"); for (i = 0; i < connections; i++) { ret = post_sends(&test.nodes[i], IBV_SEND_SIGNALED); if (ret) goto out; } ret = poll_cqs(); if (ret) goto out; printf("data transfers complete\n"); } out: rdma_destroy_id(listen_id); return ret; }
static int create_rdma(void *ctx) { // struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *id = NULL; rdma_cm_event_handler event_handler = NULL; /* if (strcmp(argv[1], "write") == 0) set_mode(M_WRITE); else if (strcmp(argv[1], "read") == 0) set_mode(M_READ); else usage(argv[0]); */ // TEST_NZ(getaddrinfo(s_ip, s_port, NULL, &addr)); struct sockaddr_in addr = { 0 }; addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(ip); addr.sin_port = htons(port); //TEST_Z(ec = rdma_create_event_channel()); TEST_NZ( id = rdma_create_id( event_handler, NULL, RDMA_PS_TCP, IB_QPT_RC ) ); //WHAT QP? TEST_NZ( rdma_resolve_addr(id, NULL, (struct sockaddr*) &addr, TIMEOUT_IN_MS) ); // kfree(addr); time_stamp(0); while ( event_handler(id, event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); // rdma_ack_cm_event(event); if (on_event(id, &event_copy)) break; } rdma_destroy_id(id); time_stamp(9); time_calculate(); return 0; }
static struct rdma_cm_id * rpcrdma_create_id(struct rpcrdma_xprt *xprt, struct rpcrdma_ia *ia, struct sockaddr *addr) { struct rdma_cm_id *id; int rc; init_completion(&ia->ri_done); id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(id)) { rc = PTR_ERR(id); dprintk("RPC: %s: rdma_create_id() failed %i\n", __func__, rc); return id; } ia->ri_async_rc = -ETIMEDOUT; rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); if (rc) { dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", __func__, rc); goto out; } wait_for_completion_interruptible_timeout(&ia->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); rc = ia->ri_async_rc; if (rc) goto out; ia->ri_async_rc = -ETIMEDOUT; rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); if (rc) { dprintk("RPC: %s: rdma_resolve_route() failed %i\n", __func__, rc); goto out; } wait_for_completion_interruptible_timeout(&ia->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); rc = ia->ri_async_rc; if (rc) goto out; return id; out: rdma_destroy_id(id); return ERR_PTR(rc); }
static int rds_rdma_listen_init(void) { struct sockaddr_in sin; struct rdma_cm_id *cm_id; int ret; cm_id = rdma_create_id(&init_net, rds_rdma_cm_event_handler, NULL, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(cm_id)) { ret = PTR_ERR(cm_id); printk(KERN_ERR "RDS/RDMA: failed to setup listener, " "rdma_create_id() returned %d\n", ret); return ret; } sin.sin_family = AF_INET; sin.sin_addr.s_addr = (__force u32)htonl(INADDR_ANY); sin.sin_port = (__force u16)htons(RDS_PORT); /* * XXX I bet this binds the cm_id to a device. If we want to support * fail-over we'll have to take this into consideration. */ ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); if (ret) { printk(KERN_ERR "RDS/RDMA: failed to setup listener, " "rdma_bind_addr() returned %d\n", ret); goto out; } ret = rdma_listen(cm_id, 128); if (ret) { printk(KERN_ERR "RDS/RDMA: failed to setup listener, " "rdma_listen() returned %d\n", ret); goto out; } rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT); rds_rdma_listen_id = cm_id; cm_id = NULL; out: if (cm_id) rdma_destroy_id(cm_id); return ret; }
void connect_to_server() { /// recv addr and port from server int server_rank = cfio_map_get_server_of_client(rank); char server_ip[NI_MAXHOST]; unsigned short server_port_int; char server_port[11]; MPI_Status sta; MPI_Recv(server_ip, NI_MAXHOST, MPI_CHAR, server_rank, rank, MPI_COMM_WORLD, &sta); MPI_Recv(&server_port_int, 1, MPI_UNSIGNED_SHORT, server_rank, rank + 1, MPI_COMM_WORLD, &sta); sprintf(server_port, "%hu", server_port_int); struct addrinfo *addr; TEST_NZ(getaddrinfo(server_ip, server_port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &cm_id, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); }
IBConnection::IBConnection(struct rdma_event_channel* ec, uint_fast16_t connection_index, uint_fast16_t remote_connection_index, struct rdma_cm_id* id) : index_(connection_index), remote_index_(remote_connection_index), cm_id_(id) { if (!cm_id_) { int err = rdma_create_id(ec, &cm_id_, this, RDMA_PS_TCP); if (err) throw InfinibandException("rdma_create_id failed"); } else { cm_id_->context = this; } qp_cap_.max_send_wr = 16; qp_cap_.max_recv_wr = 16; qp_cap_.max_send_sge = 8; qp_cap_.max_recv_sge = 8; qp_cap_.max_inline_data = 0; }
diod_rdma_t diod_rdma_create (void) { int n; diod_rdma_t rdma; rdma = malloc (sizeof (*rdma)); if (!rdma) msg_exit ("out of memory"); rdma->event_channel = rdma_create_event_channel(); if (!rdma->event_channel) msg_exit ("rdma_create_event_channel failed"); n = rdma_create_id(rdma->event_channel, &rdma->listen_id, NULL, RDMA_PS_TCP); if (n) errn_exit (n, "rdma_create_id"); return rdma; }
void accept(unsigned short port, unsigned int count) { conn_.resize(count); L_(debug) << "Setting up RDMA CM structures"; // Create rdma id (for listening) int err = rdma_create_id(ec_, &listen_id_, nullptr, RDMA_PS_TCP); if (err) { L_(error) << "rdma_create_id() failed"; throw InfinibandException("id creation failed"); } // Bind rdma id (for listening) to socket address (local port) struct sockaddr_in sin; memset(&sin, 0, sizeof sin); sin.sin_family = AF_INET; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" sin.sin_port = htons(port); sin.sin_addr.s_addr = INADDR_ANY; #pragma GCC diagnostic pop err = rdma_bind_addr(listen_id_, reinterpret_cast<struct sockaddr*>(&sin)); if (err) { L_(error) << "rdma_bind_addr(port=" << port << ") failed: " << strerror(errno); throw InfinibandException("RDMA bind_addr failed"); } // Listen for connection request on rdma id err = rdma_listen(listen_id_, count); if (err) { L_(error) << "rdma_listen() failed"; throw InfinibandException("RDMA listen failed"); } L_(debug) << "waiting for " << count << " connections"; }
int main(int argc, char **argv) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; if (argc != 4) usage(argv[0]); if (strcmp(argv[1], "write") == 0) set_mode(M_WRITE); else if (strcmp(argv[1], "read") == 0) set_mode(M_READ); else usage(argv[0]); TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) break; } rdma_destroy_event_channel(ec); return 0; }
int ibrdma_transfer(struct transfer_info tfi, int num_tfi) { struct addrinfo *addr; struct rdma_cm_id *cmid= NULL; struct rdma_event_channel *ec = NULL; struct rdma_conn_param cm_params; int i,j; /*Allocation buffer space for reading from local fs to memory*/ struct transfer_file *ffile = tfi.tfiles; int nf = tfi.tfiles; char* host = tfi.ib_host; char* port; sprintf(port,"%d",tfi.ib_port); for (i = 0; i < NUM_FILE_BUF_C; i++) { tfi.fbufs[i].fbuf = (char *)malloc(FILE_BUF_SIZE_C); tfi.fbufs[i].size = 0; } TEST_NZ(getaddrinfo(host, port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED)); freeaddrinfo(addr); build_connection(cmid); TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED)); build_params(&cm_params); TEST_NZ(rdma_connect(cmid, &cm_params)); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED)); on_connect(cmid->context); TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED)); rdma_destroy_id(&cmid); rdma_destroy_event_channel(&ec); return 0; }
void client_test(char *ip, char *port) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; TEST_NZ(getaddrinfo(ip, port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) { s_ctx->ec = ec; s_ctx->id = conn; on_connection(event_copy.id);//send our memory information to server using post_send //printf("wait for msg_send_completion\n"); poll_cq(NULL);//wait for send_completion //printf("wait for msg_recv_completion\n"); poll_cq(NULL);//wait for recv_completion break; } } return; };
void bind( boost::asio::ip::tcp::endpoint const & ep , boost::system::error_code &ec) { if(event_channel_) { HPX_IBVERBS_THROWS_IF(ec, boost::asio::error::already_connected); } else { event_channel_ = rdma_create_event_channel(); if(!event_channel_) { int verrno = errno; close(ec); boost::system::error_code err(verrno, boost::system::system_category()); HPX_IBVERBS_THROWS_IF( ec , err ); return; } set_nonblocking(event_channel_->fd, ec); if(ec) { close(ec); return; } int ret = 0; ret = rdma_create_id(event_channel_, &listener_, NULL, RDMA_PS_TCP); if(ret) { int verrno = errno; close(ec); boost::system::error_code err(verrno, boost::system::system_category()); HPX_IBVERBS_THROWS_IF( ec , err ); return; } std::string host = ep.address().to_string(); std::string port = boost::lexical_cast<std::string>(ep.port()); addrinfo *addr; getaddrinfo(host.c_str(), port.c_str(), NULL, &addr); ret = rdma_bind_addr(listener_, addr->ai_addr); freeaddrinfo(addr); if(ret) { int verrno = errno; close(ec); boost::system::error_code err(verrno, boost::system::system_category()); HPX_IBVERBS_THROWS_IF( ec , err ); return; } ret = rdma_listen(listener_, 10); /* backlog = 10 is arbitrary */ if(ret) { int verrno = errno; close(ec); boost::system::error_code err(verrno, boost::system::system_category()); HPX_IBVERBS_THROWS_IF( ec , err ); return; } HPX_IBVERBS_RESET_EC(ec); } }
int main(int argc, char *argv[]) { struct rping_cb *cb; int op; int ret = 0; int persistent_server = 0; cb = malloc(sizeof(*cb)); if (!cb) return -ENOMEM; memset(cb, 0, sizeof(*cb)); cb->server = -1; cb->state = IDLE; cb->size = 64; cb->sin.ss_family = PF_INET; cb->port = htons(7174); sem_init(&cb->sem, 0, 0); opterr = 0; while ((op=getopt(argc, argv, "a:Pp:C:S:t:scvVd")) != -1) { switch (op) { case 'a': ret = get_addr(optarg, (struct sockaddr *) &cb->sin); break; case 'P': persistent_server = 1; break; case 'p': cb->port = htons(atoi(optarg)); DEBUG_LOG("port %d\n", (int) atoi(optarg)); break; case 's': cb->server = 1; DEBUG_LOG("server\n"); break; case 'c': cb->server = 0; DEBUG_LOG("client\n"); break; case 'S': cb->size = atoi(optarg); if ((cb->size < RPING_MIN_BUFSIZE) || (cb->size > (RPING_BUFSIZE - 1))) { fprintf(stderr, "Invalid size %d " "(valid range is %Zd to %d)\n", cb->size, RPING_MIN_BUFSIZE, RPING_BUFSIZE); ret = EINVAL; } else DEBUG_LOG("size %d\n", (int) atoi(optarg)); break; case 'C': cb->count = atoi(optarg); if (cb->count < 0) { fprintf(stderr, "Invalid count %d\n", cb->count); ret = EINVAL; } else DEBUG_LOG("count %d\n", (int) cb->count); break; case 'v': cb->verbose++; DEBUG_LOG("verbose\n"); break; case 'V': cb->validate++; DEBUG_LOG("validate data\n"); break; case 'd': debug++; break; default: usage("rping"); ret = EINVAL; goto out; } } if (ret) goto out; if (cb->server == -1) { usage("rping"); ret = EINVAL; goto out; } cb->cm_channel = rdma_create_event_channel(); if (!cb->cm_channel) { perror("rdma_create_event_channel"); ret = errno; goto out; } ret = rdma_create_id(cb->cm_channel, &cb->cm_id, cb, RDMA_PS_TCP); if (ret) { perror("rdma_create_id"); goto out2; } DEBUG_LOG("created cm_id %p\n", cb->cm_id); ret = pthread_create(&cb->cmthread, NULL, cm_thread, cb); if (ret) { perror("pthread_create"); goto out2; } if (cb->server) { if (persistent_server) ret = rping_run_persistent_server(cb); else ret = rping_run_server(cb); } else { ret = rping_run_client(cb); } DEBUG_LOG("destroy cm_id %p\n", cb->cm_id); rdma_destroy_id(cb->cm_id); out2: rdma_destroy_event_channel(cb->cm_channel); out: free(cb); return ret; }
struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr, void *ctx_userdata, ibw_connstate_fn_t ibw_connstate, ibw_receive_fn_t ibw_receive, struct tevent_context *ectx) { struct ibw_ctx *ctx = talloc_zero(NULL, struct ibw_ctx); struct ibw_ctx_priv *pctx; int rc; DEBUG(DEBUG_DEBUG, ("ibw_init(ctx_userdata: %p, ectx: %p)\n", ctx_userdata, ectx)); /* initialize basic data structures */ memset(ibw_lasterr, 0, IBW_LASTERR_BUFSIZE); assert(ctx!=NULL); ibw_lasterr[0] = '\0'; talloc_set_destructor(ctx, ibw_ctx_destruct); ctx->ctx_userdata = ctx_userdata; pctx = talloc_zero(ctx, struct ibw_ctx_priv); talloc_set_destructor(pctx, ibw_ctx_priv_destruct); ctx->internal = (void *)pctx; assert(pctx!=NULL); pctx->connstate_func = ibw_connstate; pctx->receive_func = ibw_receive; pctx->ectx = ectx; /* process attributes */ if (ibw_process_init_attrs(attr, nattr, &pctx->opts)) goto cleanup; /* init cm */ pctx->cm_channel = rdma_create_event_channel(); if (!pctx->cm_channel) { sprintf(ibw_lasterr, "rdma_create_event_channel error %d\n", errno); goto cleanup; } pctx->cm_channel_event = tevent_add_fd(pctx->ectx, pctx, pctx->cm_channel->fd, TEVENT_FD_READ, ibw_event_handler_cm, ctx); #if RDMA_USER_CM_MAX_ABI_VERSION >= 2 rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx, RDMA_PS_TCP); #else rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx); #endif if (rc) { rc = errno; sprintf(ibw_lasterr, "rdma_create_id error %d\n", rc); goto cleanup; } DEBUG(DEBUG_DEBUG, ("created cm_id %p\n", pctx->cm_id)); pctx->pagesize = sysconf(_SC_PAGESIZE); return ctx; /* don't put code here */ cleanup: DEBUG(DEBUG_ERR, (ibw_lasterr)); if (ctx) talloc_free(ctx); return NULL; }
static struct rdma_cm_id * rpcrdma_create_id(struct rpcrdma_xprt *xprt, struct rpcrdma_ia *ia, struct sockaddr *addr) { struct rdma_cm_id *id; int rc; init_completion(&ia->ri_done); id = rdma_create_id(&init_net, rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(id)) { rc = PTR_ERR(id); dprintk("RPC: %s: rdma_create_id() failed %i\n", __func__, rc); return id; } ia->ri_async_rc = -ETIMEDOUT; rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); if (rc) { dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", __func__, rc); goto out; } wait_for_completion_interruptible_timeout(&ia->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); /* FIXME: * Until xprtrdma supports DEVICE_REMOVAL, the provider must * be pinned while there are active NFS/RDMA mounts to prevent * hangs and crashes at umount time. */ if (!ia->ri_async_rc && !try_module_get(id->device->owner)) { dprintk("RPC: %s: Failed to get device module\n", __func__); ia->ri_async_rc = -ENODEV; } rc = ia->ri_async_rc; if (rc) goto out; ia->ri_async_rc = -ETIMEDOUT; rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); if (rc) { dprintk("RPC: %s: rdma_resolve_route() failed %i\n", __func__, rc); goto put; } wait_for_completion_interruptible_timeout(&ia->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); rc = ia->ri_async_rc; if (rc) goto put; return id; put: module_put(id->device->owner); out: rdma_destroy_id(id); return ERR_PTR(rc); }
static int p9_rdma_bind_privport(struct p9_trans_rdma *rdma) { struct sockaddr_in cl = { .sin_family = AF_INET, .sin_addr.s_addr = htonl(INADDR_ANY), }; int port, err = -EINVAL; for (port = P9_DEF_MAX_RESVPORT; port >= P9_DEF_MIN_RESVPORT; port--) { cl.sin_port = htons((ushort)port); err = rdma_bind_addr(rdma->cm_id, (struct sockaddr *)&cl); if (err != -EADDRINUSE) break; } return err; } /** * trans_create_rdma - Transport method for creating atransport instance * @client: client instance * @addr: IP address string * @args: Mount options string */ static int rdma_create_trans(struct p9_client *client, const char *addr, char *args) { int err; struct p9_rdma_opts opts; struct p9_trans_rdma *rdma; struct rdma_conn_param conn_param; struct ib_qp_init_attr qp_attr; struct ib_device_attr devattr; struct ib_cq_init_attr cq_attr = {}; /* Parse the transport specific mount options */ err = parse_opts(args, &opts); if (err < 0) return err; /* Create and initialize the RDMA transport structure */ rdma = alloc_rdma(&opts); if (!rdma) return -ENOMEM; /* Create the RDMA CM ID */ rdma->cm_id = rdma_create_id(p9_cm_event_handler, client, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(rdma->cm_id)) goto error; /* Associate the client with the transport */ client->trans = rdma; /* Bind to a privileged port if we need to */ if (opts.privport) { err = p9_rdma_bind_privport(rdma); if (err < 0) { pr_err("%s (%d): problem binding to privport: %d\n", __func__, task_pid_nr(current), -err); goto error; } } /* Resolve the server's address */ rdma->addr.sin_family = AF_INET; rdma->addr.sin_addr.s_addr = in_aton(addr); rdma->addr.sin_port = htons(opts.port); err = rdma_resolve_addr(rdma->cm_id, NULL, (struct sockaddr *)&rdma->addr, rdma->timeout); if (err) goto error; err = wait_for_completion_interruptible(&rdma->cm_done); if (err || (rdma->state != P9_RDMA_ADDR_RESOLVED)) goto error; /* Resolve the route to the server */ err = rdma_resolve_route(rdma->cm_id, rdma->timeout); if (err) goto error; err = wait_for_completion_interruptible(&rdma->cm_done); if (err || (rdma->state != P9_RDMA_ROUTE_RESOLVED)) goto error; /* Query the device attributes */ err = ib_query_device(rdma->cm_id->device, &devattr); if (err) goto error; /* Create the Completion Queue */ cq_attr.cqe = opts.sq_depth + opts.rq_depth + 1; rdma->cq = ib_create_cq(rdma->cm_id->device, cq_comp_handler, cq_event_handler, client, &cq_attr); if (IS_ERR(rdma->cq)) goto error; ib_req_notify_cq(rdma->cq, IB_CQ_NEXT_COMP); /* Create the Protection Domain */ rdma->pd = ib_alloc_pd(rdma->cm_id->device); if (IS_ERR(rdma->pd)) goto error; /* Cache the DMA lkey in the transport */ rdma->dma_mr = NULL; if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) rdma->lkey = rdma->cm_id->device->local_dma_lkey; else { rdma->dma_mr = ib_get_dma_mr(rdma->pd, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(rdma->dma_mr)) goto error; rdma->lkey = rdma->dma_mr->lkey; } /* Create the Queue Pair */ memset(&qp_attr, 0, sizeof qp_attr); qp_attr.event_handler = qp_event_handler; qp_attr.qp_context = client; qp_attr.cap.max_send_wr = opts.sq_depth; qp_attr.cap.max_recv_wr = opts.rq_depth; qp_attr.cap.max_send_sge = P9_RDMA_SEND_SGE; qp_attr.cap.max_recv_sge = P9_RDMA_RECV_SGE; qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; qp_attr.qp_type = IB_QPT_RC; qp_attr.send_cq = rdma->cq; qp_attr.recv_cq = rdma->cq; err = rdma_create_qp(rdma->cm_id, rdma->pd, &qp_attr); if (err) goto error; rdma->qp = rdma->cm_id->qp; /* Request a connection */ memset(&conn_param, 0, sizeof(conn_param)); conn_param.private_data = NULL; conn_param.private_data_len = 0; conn_param.responder_resources = P9_RDMA_IRD; conn_param.initiator_depth = P9_RDMA_ORD; err = rdma_connect(rdma->cm_id, &conn_param); if (err) goto error; err = wait_for_completion_interruptible(&rdma->cm_done); if (err || (rdma->state != P9_RDMA_CONNECTED)) goto error; client->status = Connected; return 0; error: rdma_destroy_trans(rdma); return -ENOTCONN; }