int connect_ep(char *hostname)
{
DAT_IA_ADDRESS_PTR remote_addr = (DAT_IA_ADDRESS_PTR)&remote;
DAT_EP_ATTR ep_attr;
DAT_IA_ATTR ia_attr;
DAT_RETURN status;
DAT_REGION_DESCRIPTION region;
DAT_EVENT event;
DAT_COUNT nmore;
DAT_LMR_TRIPLET iov;
DAT_RMR_TRIPLET *r_iov;
DAT_DTO_COOKIE cookie;
DAT_CONN_QUAL conn_qual;
DAT_BOOLEAN in, out;
int i, ii, pdata, ctx;
DAT_PROVIDER_ATTR prov_attrs;
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
status = dat_ia_open(provider, 8, &async_evd, &ia);
_OK(status, "dat_ia_open");
memset(&prov_attrs, 0, sizeof(prov_attrs));
status = dat_ia_query(ia, NULL,
DAT_IA_FIELD_ALL, &ia_attr,
DAT_PROVIDER_FIELD_ALL, &prov_attrs);
_OK(status, "dat_ia_query");
print_ia_address(ia_attr.ia_address_ptr);
if (ucm && ud_test) {
printf("%d UD test over UCM provider not supported\n",
getpid());
exit(1);
}
/* Print provider specific attributes */
for (i = 0; i < prov_attrs.num_provider_specific_attr; i++) {
LOGPRINTF(" Provider Specific Attribute[%d] %s=%s\n",
i, prov_attrs.provider_specific_attr[i].name,
prov_attrs.provider_specific_attr[i].value);
/* check for counter support */
status = strcmp(prov_attrs.provider_specific_attr[i].name,
"DAT_COUNTERS");
if (!status)
counters_ok = 1;
}
/* make sure provider supports counters */
if ((counters) && (!counters_ok)) {
printf("Disable dat_query_counters:"
" Provider not built with counters\n");
counters = 0;
}
status = dat_pz_create(ia, &pz);
_OK(status, "dat_pz_create");
status = dat_evd_create(ia, eps * 2, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
&cr_evd);
_OK(status, "dat_evd_create CR");
status = dat_evd_create(ia, eps * 2, DAT_HANDLE_NULL,
DAT_EVD_CONNECTION_FLAG, &con_evd);
_OK(status, "dat_evd_create CR");
status = dat_evd_create(ia, eps * 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG,
&dto_evd);
_OK(status, "dat_evd_create DTO");
memset(&ep_attr, 0, sizeof(ep_attr));
if (ud_test) {
msg_size += 40;
ep_attr.service_type = DAT_IB_SERVICE_TYPE_UD;
ep_attr.max_message_size = buf_size;
ep_attr.max_rdma_read_in = 0;
ep_attr.max_rdma_read_out = 0;
} else {
ep_attr.service_type = DAT_SERVICE_TYPE_RC;
ep_attr.max_rdma_size = 0x10000;
ep_attr.max_rdma_read_in = 4;
ep_attr.max_rdma_read_out = 4;
}
ep_attr.qos = 0;
ep_attr.recv_completion_flags = 0;
ep_attr.max_recv_dtos = eps * 10;
ep_attr.max_request_dtos = eps * 10;
ep_attr.max_recv_iov = 1;
ep_attr.max_request_iov = 1;
ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
ep_attr.ep_transport_specific_count = 0;
ep_attr.ep_transport_specific = NULL;
ep_attr.ep_provider_specific_count = 0;
ep_attr.ep_provider_specific = NULL;
for (i = 0; i < eps; i++) {
status = dat_ep_create(ia, pz, dto_evd, dto_evd,
con_evd, &ep_attr, &ep[i]);
_OK(status, "dat_ep_create");
LOGPRINTF(" create_ep[%d]=%p\n", i, ep[i]);
}
for (i = 0; i < REG_MEM_COUNT * eps; i++) {
buf[i] = (DAT_RMR_TRIPLET *) malloc(buf_size);
region.for_va = buf[i];
status = dat_lmr_create(ia,
DAT_MEM_TYPE_VIRTUAL,
region,
buf_size,
pz,
DAT_MEM_PRIV_ALL_FLAG |
DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
&lmr[i],
&lmr_context[i],
&rmr_context[i],
®_size[i], ®_addr[i]);
_OK(status, "dat_lmr_create");
}
/* register atomic return buffer for original data */
atomic_buf = (DAT_UINT64 *) malloc(BUF_SIZE_ATOMIC);
region.for_va = atomic_buf;
status = dat_lmr_create(ia,
DAT_MEM_TYPE_VIRTUAL,
region,
BUF_SIZE_ATOMIC,
pz,
DAT_MEM_PRIV_ALL_FLAG |
DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
&lmr_atomic,
&lmr_atomic_context,
&rmr_atomic_context,
®_atomic_size, ®_atomic_addr);
_OK(status, "dat_lmr_create atomic");
for (ii = 0; ii < eps; ii++) {
for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
int ep_idx = 0;
cookie.as_64 = (ii * REG_MEM_COUNT) + i;
iov.lmr_context = lmr_context[(ii * REG_MEM_COUNT) + i];
iov.virtual_address =
(DAT_VADDR) (uintptr_t) buf[(ii * REG_MEM_COUNT) +
i];
iov.segment_length = buf_size;
LOGPRINTF(" post_recv (%p) on ep[%d]=%p\n",
buf[(ii * REG_MEM_COUNT) + i], ii, ep[ii]);
/* ep[0], unless testing Server and multi EP's */
if (server && multi_eps) {
ep_idx = ii;
cookie.as_64 = i;
}
status = dat_ep_post_recv(ep[ep_idx],
1,
&iov,
cookie,
DAT_COMPLETION_DEFAULT_FLAG);
_OK(status, "dat_ep_post_recv");
}
}
/* setup receive buffer to initial string to be overwritten */
strcpy((char *)buf[RCV_RDMA_BUF_INDEX], "blah, blah, blah\n");
/* ud can resolve_ah and connect both ways, same EP */
if (server || (!server && ud_test)) {
if (server) {
conn_qual = SERVER_ID;
strcpy((char *)buf[SND_RDMA_BUF_INDEX], "Server data");
} else {
conn_qual = CLIENT_ID;
strcpy((char *)buf[SND_RDMA_BUF_INDEX], "Client data");
}
status = dat_psp_create(ia,
conn_qual,
cr_evd, DAT_PSP_CONSUMER_FLAG, &psp);
_OK(status, "dat_psp_create");
/* Server always waits for first CR from Client */
if (server)
process_cr(0);
}
/* ud can resolve_ah and connect both ways */
if (!server || (server && ud_test)) {
struct addrinfo *target;
if (ucm)
goto no_resolution;
if (getaddrinfo(hostname, NULL, NULL, &target) != 0) {
printf("Error getting remote address.\n");
exit(1);
}
printf("Remote %s Name: %s \n",
server ? "Client" : "Server", hostname);
printf("Remote %s Net Address: %s\n",
server ? "Client" : "Server",
inet_ntoa(((struct sockaddr_in *)
target->ai_addr)->sin_addr));
strcpy((char *)buf[SND_RDMA_BUF_INDEX], "Client written data");
remote_addr = (DAT_IA_ADDRESS_PTR)target->ai_addr; /* IP */
no_resolution:
/* one Client EP, multiple Server EPs, same conn_qual
* use private data to select EP on Server
*/
for (i = 0; i < eps; i++) {
/* pdata selects Server EP,
* support both muliple Server and single EP's
*/
if (multi_eps)
pdata = hton32(i);
else
pdata = 0; /* just use first EP */
status = dat_ep_connect(ep[0],
remote_addr,
(server ? CLIENT_ID :
SERVER_ID), CONN_TIMEOUT, 4,
(DAT_PVOID) & pdata, 0,
DAT_CONNECT_DEFAULT_FLAG);
_OK(status, "dat_ep_connect");
}
if (!ucm)
freeaddrinfo(target);
}
/* UD: process CR's starting with 2nd on server, 1st for client */
if (ud_test) {
for (i = (server ? 1 : 0); i < eps; i++)
process_cr(i);
}
/* RC and UD: process CONN EST events */
for (i = 0; i < eps; i++)
process_conn(i);
/* UD: CONN EST events for CONN's and CR's */
if (ud_test) {
for (i = 0; i < eps; i++)
process_conn(i);
}
printf("Connected! %d endpoints\n", eps);
/*
* Setup our remote memory and tell the other side about it
* Swap to network order.
*/
r_iov = (DAT_RMR_TRIPLET *) buf[SEND_BUF_INDEX];
r_iov->rmr_context = hton32(rmr_context[RCV_RDMA_BUF_INDEX]);
r_iov->virtual_address =
hton64((DAT_VADDR) (uintptr_t) buf[RCV_RDMA_BUF_INDEX]);
r_iov->segment_length = hton32(buf_size);
printf("Send RMR message: r_key_ctx=0x%x,va=" F64x ",len=0x%x\n",
hton32(r_iov->rmr_context),
hton64(r_iov->virtual_address), hton32(r_iov->segment_length));
send_msg(buf[SEND_BUF_INDEX],
sizeof(DAT_RMR_TRIPLET),
lmr_context[SEND_BUF_INDEX],
cookie, DAT_COMPLETION_SUPPRESS_FLAG);
dat_ep_get_status(ep[0], NULL, &in, &out);
printf("EP[0] status: posted buffers: Req=%d, Rcv=%d\n", in, out);
/*
* Wait for their RMR
*/
for (i = 0, ctx = 0; i < eps; i++, ctx++) {
/* expected cookie, recv buf idx in every mem pool */
ctx = (ctx % REG_MEM_COUNT) ? ctx : ctx + RECV_BUF_INDEX;
LOGPRINTF("Waiting for remote to send RMR data\n");
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait after dat_ep_post_send");
if ((event.event_number != DAT_DTO_COMPLETION_EVENT) &&
(ud_test && event.event_number != DAT_IB_DTO_EVENT)) {
printf("unexpected event waiting for RMR context "
"- 0x%x\n", event.event_number);
exit(1);
}
_OK(dto_event->status, "event status for post_recv");
/* careful when checking cookies:
* Client - receiving multi messages on a single EP
* Server - not receiving on multiple EP's
*/
if (!server || (server && !multi_eps)) {
if (dto_event->transfered_length != msg_size ||
dto_event->user_cookie.as_64 != ctx) {
printf("unexpected event data on recv: len=%d"
" cookie=" F64x " expected %d/%d\n",
(int)dto_event->transfered_length,
dto_event->user_cookie.as_64,
msg_size, ctx);
exit(1);
}
/* Server - receiving one message each across many EP's */
} else {
if (dto_event->transfered_length != msg_size ||
dto_event->user_cookie.as_64 != RECV_BUF_INDEX) {
printf("unexpected event data on recv: len=%d"
"cookie=" F64x " expected %d/%d\n",
(int)dto_event->transfered_length,
dto_event->user_cookie.as_64,
msg_size, RECV_BUF_INDEX);
exit(1);
}
}
/* swap RMR,address info to host order */
if (!server || (server && !multi_eps))
r_iov = (DAT_RMR_TRIPLET *) buf[ctx];
else
r_iov =
(DAT_RMR_TRIPLET *) buf[(i * REG_MEM_COUNT) +
RECV_BUF_INDEX];
if (ud_test)
r_iov = (DAT_RMR_TRIPLET *) ((char *)r_iov + 40);
r_iov->rmr_context = ntoh32(r_iov->rmr_context);
r_iov->virtual_address = ntoh64(r_iov->virtual_address);
r_iov->segment_length = ntoh32(r_iov->segment_length);
printf("Recv RMR message: r_iov(%p):"
" r_key_ctx=%x,va=" F64x ",len=0x%x on EP=%p\n",
r_iov, r_iov->rmr_context,
r_iov->virtual_address,
r_iov->segment_length, dto_event->ep_handle);
}
return (0);
}
/* Connect a remote PSP at addr : conn_qual
* return -1 on error */
int psdapl_connect(psdapl_con_info_t *ci, psdapl_info_msg_t *msg)
{
int rc;
DAT_RETURN dat_rc;
rc = psdapl_init_buffers_local(ci);
if (rc) goto err_init_buf;
rc = psdapl_create_ep(ci);
if (rc) goto err_create_ep;
psdapl_init_msg_t res_imsg;
psdapl_init_init_msg(&res_imsg, ci);
dat_rc = dat_ep_connect(ci->ep_handle,
&msg->sock_addr,
msg->conn_qual,
DAT_TIMEOUT_INFINITE /* 5 * 1000 * 1000 */,
sizeof(res_imsg) /* private_data_size */,
&res_imsg /* private_data */,
DAT_QOS_BEST_EFFORT /* DAT_QOS */,
DAT_CONNECT_DEFAULT_FLAG /* DAT_CONNECT_FLAGS */);
if (dat_rc != DAT_SUCCESS) goto err_ep_connect;
DAT_EVENT event;
DAT_COUNT nmore;
event.event_number = -1;
dat_rc = dat_evd_wait(ci->connect_evd_handle,
DAT_TIMEOUT_INFINITE /* 5*1000*1000 timeout in usec*/,
1 /* threshold */,
&event, &nmore);
psdapl_init_msg_t *imsg = NULL;
switch (DAT_GET_TYPE(dat_rc)) {
/*
case DAT_TIMEOUT_EXPIRED:
fprintf(stderr, "<mark (timeout)>\n");
break;
*/
case DAT_SUCCESS:
switch (event.event_number) {
case DAT_CONNECTION_EVENT_TIMED_OUT:
psdapl_dprint(2, "psdapl_connect: event DAT_CONNECTION_EVENT_TIMED_OUT");
break;
case DAT_CONNECTION_EVENT_ESTABLISHED:
psdapl_dprint(3, "psdapl_connect: event DAT_CONNECTION_EVENT_ESTABLISHED");
DAT_CONNECTION_EVENT_DATA *cd = &event.event_data.connect_event_data;
imsg = (psdapl_init_msg_t *)(cd->private_data);
break;
default:
psdapl_dprint(2, "psdapl_connect: unexpected event 0x%x",
(unsigned)event.event_number);
break;
}
break;
default:
psdapl_dprint_dat_err(1, dat_rc, "psdapl_connect: dat_evd_wait()");
break;
}
if (!imsg) goto err_wait;
psdapl_init_buffers_remote(ci, imsg);
return 0;
/* --- */
err_ep_connect:
psdapl_dprint_dat_err(0, dat_rc, "dat_ep_connect() failed");
goto err_all;
/* --- */
err_all:
err_wait:
err_create_ep:
err_init_buf:
/* ToDo: Cleanup recv_evd_handle!! */
/* ToDo: Cleanup connect_evd_handle!! */
/* ToDo: Cleanup bufpairs!!!!! */
return -1;
}
