/* Wait for a disconnected event.
*
* return -1 on error */
static
int psdapl_wait4event(psdapl_con_info_t *ci, unsigned ev, const char *ev_name)
{
DAT_EVENT event;
DAT_COUNT nmore;
DAT_RETURN dat_rc;
dat_rc = dat_evd_wait(ci->connect_evd_handle,
3*1000*1000 /* timeout in usec */,
1 /* threshold */,
&event, &nmore);
switch (DAT_GET_TYPE(dat_rc)) {
case DAT_TIMEOUT_EXPIRED:
psdapl_dprint(2, "psdapl_wait4event(%s): DAT_TIMEOUT_EXPIRED",
ev_name);
return -1;
break;
case DAT_SUCCESS:
if (event.event_number == ev) {
psdapl_dprint(3, "event %s", ev_name);
return 0;
} else {
psdapl_dprint(2, "psdapl_wait4event(%s): unexpected event 0x%x",
ev_name, (unsigned)event.event_number);
}
break;
default:
psdapl_dprint_dat_err(2, dat_rc, "psdapl_wait4event(%s): dat_evd_wait()",
ev_name);
}
return -1;
}
/* read all events from recv_evd_handle */
static
void psdapl_flush_evd(psdapl_con_info_t *ci)
{
while (1) {
DAT_RETURN dat_rc;
DAT_EVENT event;
DAT_COUNT nmore = 0;
#if 0
dat_rc = dat_evd_wait(ci->recv_evd_handle,
0 /*timeout in usec*/,
1 /* threshold */,
&event, &nmore);
#else
dat_rc = dat_evd_dequeue(ci->recv_evd_handle, &event);
nmore = 1;
#endif
switch (DAT_GET_TYPE(dat_rc)) {
case DAT_TIMEOUT_EXPIRED:
// psdapl_dprint(3, "psdapl_flush_evd event DAT_TIMEOUT_EXPIRED. nmore:%d", nmore);
ci->outstanding_cq_entries = 0;
psdapl_stat.timeouts++;
break;
case DAT_SUCCESS:
switch (event.event_number) {
case DAT_DTO_COMPLETION_EVENT:
if (!event.event_data.dto_completion_event_data.user_cookie.as_ptr) {
// From sendv
if (ci->outstanding_cq_entries) {
ci->outstanding_cq_entries--;
}
} else {
do_DTO_COMPLETION_EVENT(ci, &event.event_data.dto_completion_event_data);
}
// psdapl_dprint(3, "psdapl_flush_evd event DAT_DTO_COMPLETION_EVENT. nmore:%d", nmore);
break;
default:
psdapl_dprint(1, "psdapl_flush_evd: unexpected event 0x%x. nmore:%d",
(unsigned)event.event_number, nmore);
break;
}
break;
case DAT_QUEUE_EMPTY:
nmore = 0;
break;
default:
nmore = 0;
psdapl_dprint_dat_err(1, dat_rc, "psdapl_flush_evd: dat_evd_wait(). nmore:%d",
nmore);
}
if (!nmore) break;
}
}
/* RC - Server only, UD - Server and Client, one per EP */
void process_cr(int idx)
{
DAT_EVENT event;
DAT_COUNT nmore;
DAT_RETURN status;
int pdata;
DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
DAT_CONN_QUAL exp_qual = server ? SERVER_ID : CLIENT_ID;
DAT_CR_PARAM cr_param;
DAT_CR_ARRIVAL_EVENT_DATA *cr_event =
&event.event_data.cr_arrival_event_data;
LOGPRINTF("%s waiting for connect[%d] request\n",
server ? "Server" : "Client", idx);
status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
_OK(status, "CR dat_evd_wait");
if (event.event_number != DAT_CONNECTION_REQUEST_EVENT &&
(ud_test && event.event_number !=
DAT_IB_UD_CONNECTION_REQUEST_EVENT)) {
printf("unexpected event,!conn req: 0x%x\n",
event.event_number);
exit(1);
}
if ((cr_event->conn_qual != exp_qual) ||
(cr_event->sp_handle.psp_handle != psp)) {
printf("wrong cr event data\n");
exit(1);
}
cr = cr_event->cr_handle;
status = dat_cr_query(cr, DAT_CSP_FIELD_ALL, &cr_param);
_OK(status, "dat_cr_query");
/* use private data to select EP */
pdata = ntoh32(*((int *)cr_param.private_data));
LOGPRINTF("%s recvd pdata=0x%x, send pdata=0x%x\n",
server ? "Server" : "Client", pdata,
*(int *)cr_param.private_data);
status = dat_cr_accept(cr, ep[pdata], 4, cr_param.private_data);
_OK(status, "dat_cr_accept");
printf("%s accepted CR on EP[%d]=%p\n",
server ? "Server" : "Client", pdata, ep[pdata]);
}
int do_immediate()
{
DAT_EVENT event;
DAT_COUNT nmore;
DAT_LMR_TRIPLET iov;
DAT_RMR_TRIPLET r_iov;
DAT_DTO_COOKIE cookie;
DAT_RETURN status;
DAT_UINT32 immed_data;
DAT_UINT32 immed_data_recv = 0;
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
DAT_IB_EXTENSION_EVENT_DATA *ext_event =
(DAT_IB_EXTENSION_EVENT_DATA *) & event.event_extension_data[0];
printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");
if (server) {
immed_data = 0x1111;
} else {
immed_data = 0x7777;
}
cookie.as_64 = 0x5555;
/* RMR info already swapped back to host order in connect_ep */
r_iov = *buf[RECV_BUF_INDEX];
iov.lmr_context = lmr_context[SND_RDMA_BUF_INDEX];
iov.virtual_address = (DAT_VADDR) (uintptr_t) buf[SND_RDMA_BUF_INDEX];
iov.segment_length = buf_size;
cookie.as_64 = 0x9999;
status = dat_ib_post_rdma_write_immed(ep[0], // ep_handle
1, // segments
&iov, // LMR
cookie, // user_cookie
&r_iov, // RMR
immed_data,
DAT_COMPLETION_DEFAULT_FLAG);
_OK(status, "dat_ib_post_rdma_write_immed");
/*
* Collect first event, write completion or inbound recv with immed
*/
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
if (event.event_number != DAT_IB_DTO_EVENT) {
printf("unexpected event #0x%x waiting for WR-IMMED #0x%x\n",
event.event_number, DAT_IB_DTO_EVENT);
exit(1);
}
if (nmore)
printf("%s() nmore %d\n", __FUNCTION__, nmore);
_OK(dto_event->status, "DTO event status");
if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED) {
if ((dto_event->transfered_length != buf_size) ||
(dto_event->user_cookie.as_64 != 0x9999)) {
printf
("unexpected event data for rdma_write_immed: len=%d "
"cookie=0x%x\n", (int)dto_event->transfered_length,
(int)dto_event->user_cookie.as_64);
exit(1);
}
} else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA) {
if ((dto_event->transfered_length != buf_size) ||
(dto_event->user_cookie.as_64 != RECV_BUF_INDEX + 1)) {
printf
("unexpected event data of immediate write: len=%d "
"cookie=" F64x " expected %d/%d\n",
(int)dto_event->transfered_length,
dto_event->user_cookie.as_64, (int)sizeof(int),
RECV_BUF_INDEX + 1);
exit(1);
}
/* get immediate data from event */
immed_data_recv = ext_event->val.immed.data;
} else {
printf("unexpected extension type for event - 0x%x, 0x%x\n",
event.event_number, ext_event->type);
exit(1);
}
/*
* Collect second event, write completion or inbound recv with immed
*/
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
if (event.event_number != DAT_IB_DTO_EVENT) {
printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
event.event_number);
exit(1);
}
_OK(dto_event->status, "event status");
if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED) {
if ((dto_event->transfered_length != buf_size) ||
(dto_event->user_cookie.as_64 != 0x9999)) {
printf
("unexpected event data for rdma_write_immed: len=%d "
"cookie=0x%x\n", (int)dto_event->transfered_length,
(int)dto_event->user_cookie.as_64);
exit(1);
}
} else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA) {
if ((dto_event->transfered_length != buf_size) ||
(dto_event->user_cookie.as_64 != RECV_BUF_INDEX + 1)) {
printf
("unexpected event data of immediate write: len=%d "
"cookie=" F64x " expected %d/%d\n",
(int)dto_event->transfered_length,
dto_event->user_cookie.as_64, (int)sizeof(int),
RECV_BUF_INDEX + 1);
exit(1);
}
/* get immediate data from event */
immed_data_recv = ext_event->val.immed.data;
} else {
printf("unexpected extension type for event - 0x%x, 0x%x\n",
event.event_number, ext_event->type);
exit(1);
}
if ((server) && (immed_data_recv != 0x7777)) {
printf("ERROR: Server: unexpected imm_data_recv 0x%x/0x%x\n",
0x7777, immed_data_recv);
exit(1);
} else if ((!server) && (immed_data_recv != 0x1111)) {
printf("ERROR: Client: unexpected imm_data_recv 0x%x/0x%x\n",
0x1111, immed_data_recv);
exit(1);
}
if (server)
printf("Server received immed_data=0x%x\n", immed_data_recv);
else
printf("Client received immed_data=0x%x\n", immed_data_recv);
printf("rdma buffer %p contains: %s\n",
buf[RCV_RDMA_BUF_INDEX], (char *)buf[RCV_RDMA_BUF_INDEX]);
printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");
return (0);
}
int disconnect_ep(void)
{
DAT_RETURN status;
DAT_EVENT event;
DAT_COUNT nmore;
int i;
if (counters) { /* examples of query and print */
int ii;
DAT_UINT64 ia_cntrs[DCNT_IA_ALL_COUNTERS];
dat_query_counters(ia, DCNT_IA_ALL_COUNTERS, ia_cntrs, 0);
printf(" IA Cntrs:");
for (ii = 0; ii < DCNT_IA_ALL_COUNTERS; ii++)
printf(" " F64u "", ia_cntrs[ii]);
printf("\n");
dat_print_counters(ia, DCNT_IA_ALL_COUNTERS, 0);
}
if (!ud_test) {
status = dat_ep_disconnect(ep[0], DAT_CLOSE_DEFAULT);
_OK2(status, "dat_ep_disconnect");
status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1,
&event, &nmore);
_OK(status, "dat_evd_wait");
}
if (psp) {
status = dat_psp_free(psp);
_OK2(status, "dat_psp_free");
}
for (i = 0; i < REG_MEM_COUNT * eps; i++) {
status = dat_lmr_free(lmr[i]);
_OK2(status, "dat_lmr_free");
}
if (lmr_atomic) {
status = dat_lmr_free(lmr_atomic);
_OK2(status, "dat_lmr_free_atomic");
}
for (i = 0; i < eps; i++) {
if (counters) { /* examples of query and print */
int ii;
DAT_UINT64 ep_cntrs[DCNT_EP_ALL_COUNTERS];
dat_query_counters(ep[i], DCNT_EP_ALL_COUNTERS,
ep_cntrs, 0);
printf(" EP[%d] Cntrs:", i);
for (ii = 0; ii < DCNT_EP_ALL_COUNTERS; ii++)
printf(" " F64u "", ep_cntrs[ii]);
printf("\n");
dat_print_counters(ep[i], DCNT_EP_ALL_COUNTERS, 0);
}
status = dat_ep_free(ep[i]);
_OK2(status, "dat_ep_free");
}
if (counters) { /* examples of query and print */
int ii;
DAT_UINT64 evd_cntrs[DCNT_EVD_ALL_COUNTERS];
dat_query_counters(dto_evd, DCNT_EVD_ALL_COUNTERS,
evd_cntrs, 0);
printf(" DTO_EVD Cntrs:");
for (ii = 0; ii < DCNT_EVD_ALL_COUNTERS; ii++)
printf(" " F64u "", evd_cntrs[ii]);
printf("\n");
dat_print_counters(dto_evd, DCNT_EVD_ALL_COUNTERS, 0);
dat_query_counters(con_evd, DCNT_EVD_ALL_COUNTERS,
evd_cntrs, 0);
printf(" CONN_EVD Cntrs:");
for (ii = 0; ii < DCNT_EVD_ALL_COUNTERS; ii++)
printf(" " F64u "", evd_cntrs[ii]);
printf("\n");
dat_print_counters(con_evd, DCNT_EVD_ALL_COUNTERS, 0);
dat_query_counters(cr_evd, DCNT_EVD_ALL_COUNTERS, evd_cntrs, 0);
printf(" CR_EVD Cntrs:");
for (ii = 0; ii < DCNT_EVD_ALL_COUNTERS; ii++)
printf(" " F64u "", evd_cntrs[ii]);
printf("\n");
dat_print_counters(cr_evd, DCNT_EVD_ALL_COUNTERS, 0);
}
status = dat_evd_free(dto_evd);
_OK2(status, "dat_evd_free DTO");
status = dat_evd_free(con_evd);
_OK2(status, "dat_evd_free CON");
status = dat_evd_free(cr_evd);
_OK2(status, "dat_evd_free CR");
status = dat_pz_free(pz);
_OK2(status, "dat_pz_free");
status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
_OK2(status, "dat_ia_close");
return (0);
}
int connect_ep(char *hostname)
{
DAT_IA_ADDRESS_PTR remote_addr = (DAT_IA_ADDRESS_PTR)&remote;
DAT_EP_ATTR ep_attr;
DAT_IA_ATTR ia_attr;
DAT_RETURN status;
DAT_REGION_DESCRIPTION region;
DAT_EVENT event;
DAT_COUNT nmore;
DAT_LMR_TRIPLET iov;
DAT_RMR_TRIPLET *r_iov;
DAT_DTO_COOKIE cookie;
DAT_CONN_QUAL conn_qual;
DAT_BOOLEAN in, out;
int i, ii, pdata, ctx;
DAT_PROVIDER_ATTR prov_attrs;
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
status = dat_ia_open(provider, 8, &async_evd, &ia);
_OK(status, "dat_ia_open");
memset(&prov_attrs, 0, sizeof(prov_attrs));
status = dat_ia_query(ia, NULL,
DAT_IA_FIELD_ALL, &ia_attr,
DAT_PROVIDER_FIELD_ALL, &prov_attrs);
_OK(status, "dat_ia_query");
print_ia_address(ia_attr.ia_address_ptr);
if (ucm && ud_test) {
printf("%d UD test over UCM provider not supported\n",
getpid());
exit(1);
}
/* Print provider specific attributes */
for (i = 0; i < prov_attrs.num_provider_specific_attr; i++) {
LOGPRINTF(" Provider Specific Attribute[%d] %s=%s\n",
i, prov_attrs.provider_specific_attr[i].name,
prov_attrs.provider_specific_attr[i].value);
/* check for counter support */
status = strcmp(prov_attrs.provider_specific_attr[i].name,
"DAT_COUNTERS");
if (!status)
counters_ok = 1;
}
/* make sure provider supports counters */
if ((counters) && (!counters_ok)) {
printf("Disable dat_query_counters:"
" Provider not built with counters\n");
counters = 0;
}
status = dat_pz_create(ia, &pz);
_OK(status, "dat_pz_create");
status = dat_evd_create(ia, eps * 2, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
&cr_evd);
_OK(status, "dat_evd_create CR");
status = dat_evd_create(ia, eps * 2, DAT_HANDLE_NULL,
DAT_EVD_CONNECTION_FLAG, &con_evd);
_OK(status, "dat_evd_create CR");
status = dat_evd_create(ia, eps * 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG,
&dto_evd);
_OK(status, "dat_evd_create DTO");
memset(&ep_attr, 0, sizeof(ep_attr));
if (ud_test) {
msg_size += 40;
ep_attr.service_type = DAT_IB_SERVICE_TYPE_UD;
ep_attr.max_message_size = buf_size;
ep_attr.max_rdma_read_in = 0;
ep_attr.max_rdma_read_out = 0;
} else {
ep_attr.service_type = DAT_SERVICE_TYPE_RC;
ep_attr.max_rdma_size = 0x10000;
ep_attr.max_rdma_read_in = 4;
ep_attr.max_rdma_read_out = 4;
}
ep_attr.qos = 0;
ep_attr.recv_completion_flags = 0;
ep_attr.max_recv_dtos = eps * 10;
ep_attr.max_request_dtos = eps * 10;
ep_attr.max_recv_iov = 1;
ep_attr.max_request_iov = 1;
ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
ep_attr.ep_transport_specific_count = 0;
ep_attr.ep_transport_specific = NULL;
ep_attr.ep_provider_specific_count = 0;
ep_attr.ep_provider_specific = NULL;
for (i = 0; i < eps; i++) {
status = dat_ep_create(ia, pz, dto_evd, dto_evd,
con_evd, &ep_attr, &ep[i]);
_OK(status, "dat_ep_create");
LOGPRINTF(" create_ep[%d]=%p\n", i, ep[i]);
}
for (i = 0; i < REG_MEM_COUNT * eps; i++) {
buf[i] = (DAT_RMR_TRIPLET *) malloc(buf_size);
region.for_va = buf[i];
status = dat_lmr_create(ia,
DAT_MEM_TYPE_VIRTUAL,
region,
buf_size,
pz,
DAT_MEM_PRIV_ALL_FLAG |
DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
&lmr[i],
&lmr_context[i],
&rmr_context[i],
®_size[i], ®_addr[i]);
_OK(status, "dat_lmr_create");
}
/* register atomic return buffer for original data */
atomic_buf = (DAT_UINT64 *) malloc(BUF_SIZE_ATOMIC);
region.for_va = atomic_buf;
status = dat_lmr_create(ia,
DAT_MEM_TYPE_VIRTUAL,
region,
BUF_SIZE_ATOMIC,
pz,
DAT_MEM_PRIV_ALL_FLAG |
DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
&lmr_atomic,
&lmr_atomic_context,
&rmr_atomic_context,
®_atomic_size, ®_atomic_addr);
_OK(status, "dat_lmr_create atomic");
for (ii = 0; ii < eps; ii++) {
for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
int ep_idx = 0;
cookie.as_64 = (ii * REG_MEM_COUNT) + i;
iov.lmr_context = lmr_context[(ii * REG_MEM_COUNT) + i];
iov.virtual_address =
(DAT_VADDR) (uintptr_t) buf[(ii * REG_MEM_COUNT) +
i];
iov.segment_length = buf_size;
LOGPRINTF(" post_recv (%p) on ep[%d]=%p\n",
buf[(ii * REG_MEM_COUNT) + i], ii, ep[ii]);
/* ep[0], unless testing Server and multi EP's */
if (server && multi_eps) {
ep_idx = ii;
cookie.as_64 = i;
}
status = dat_ep_post_recv(ep[ep_idx],
1,
&iov,
cookie,
DAT_COMPLETION_DEFAULT_FLAG);
_OK(status, "dat_ep_post_recv");
}
}
/* setup receive buffer to initial string to be overwritten */
strcpy((char *)buf[RCV_RDMA_BUF_INDEX], "blah, blah, blah\n");
/* ud can resolve_ah and connect both ways, same EP */
if (server || (!server && ud_test)) {
if (server) {
conn_qual = SERVER_ID;
strcpy((char *)buf[SND_RDMA_BUF_INDEX], "Server data");
} else {
conn_qual = CLIENT_ID;
strcpy((char *)buf[SND_RDMA_BUF_INDEX], "Client data");
}
status = dat_psp_create(ia,
conn_qual,
cr_evd, DAT_PSP_CONSUMER_FLAG, &psp);
_OK(status, "dat_psp_create");
/* Server always waits for first CR from Client */
if (server)
process_cr(0);
}
/* ud can resolve_ah and connect both ways */
if (!server || (server && ud_test)) {
struct addrinfo *target;
if (ucm)
goto no_resolution;
if (getaddrinfo(hostname, NULL, NULL, &target) != 0) {
printf("Error getting remote address.\n");
exit(1);
}
printf("Remote %s Name: %s \n",
server ? "Client" : "Server", hostname);
printf("Remote %s Net Address: %s\n",
server ? "Client" : "Server",
inet_ntoa(((struct sockaddr_in *)
target->ai_addr)->sin_addr));
strcpy((char *)buf[SND_RDMA_BUF_INDEX], "Client written data");
remote_addr = (DAT_IA_ADDRESS_PTR)target->ai_addr; /* IP */
no_resolution:
/* one Client EP, multiple Server EPs, same conn_qual
* use private data to select EP on Server
*/
for (i = 0; i < eps; i++) {
/* pdata selects Server EP,
* support both muliple Server and single EP's
*/
if (multi_eps)
pdata = hton32(i);
else
pdata = 0; /* just use first EP */
status = dat_ep_connect(ep[0],
remote_addr,
(server ? CLIENT_ID :
SERVER_ID), CONN_TIMEOUT, 4,
(DAT_PVOID) & pdata, 0,
DAT_CONNECT_DEFAULT_FLAG);
_OK(status, "dat_ep_connect");
}
if (!ucm)
freeaddrinfo(target);
}
/* UD: process CR's starting with 2nd on server, 1st for client */
if (ud_test) {
for (i = (server ? 1 : 0); i < eps; i++)
process_cr(i);
}
/* RC and UD: process CONN EST events */
for (i = 0; i < eps; i++)
process_conn(i);
/* UD: CONN EST events for CONN's and CR's */
if (ud_test) {
for (i = 0; i < eps; i++)
process_conn(i);
}
printf("Connected! %d endpoints\n", eps);
/*
* Setup our remote memory and tell the other side about it
* Swap to network order.
*/
r_iov = (DAT_RMR_TRIPLET *) buf[SEND_BUF_INDEX];
r_iov->rmr_context = hton32(rmr_context[RCV_RDMA_BUF_INDEX]);
r_iov->virtual_address =
hton64((DAT_VADDR) (uintptr_t) buf[RCV_RDMA_BUF_INDEX]);
r_iov->segment_length = hton32(buf_size);
printf("Send RMR message: r_key_ctx=0x%x,va=" F64x ",len=0x%x\n",
hton32(r_iov->rmr_context),
hton64(r_iov->virtual_address), hton32(r_iov->segment_length));
send_msg(buf[SEND_BUF_INDEX],
sizeof(DAT_RMR_TRIPLET),
lmr_context[SEND_BUF_INDEX],
cookie, DAT_COMPLETION_SUPPRESS_FLAG);
dat_ep_get_status(ep[0], NULL, &in, &out);
printf("EP[0] status: posted buffers: Req=%d, Rcv=%d\n", in, out);
/*
* Wait for their RMR
*/
for (i = 0, ctx = 0; i < eps; i++, ctx++) {
/* expected cookie, recv buf idx in every mem pool */
ctx = (ctx % REG_MEM_COUNT) ? ctx : ctx + RECV_BUF_INDEX;
LOGPRINTF("Waiting for remote to send RMR data\n");
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait after dat_ep_post_send");
if ((event.event_number != DAT_DTO_COMPLETION_EVENT) &&
(ud_test && event.event_number != DAT_IB_DTO_EVENT)) {
printf("unexpected event waiting for RMR context "
"- 0x%x\n", event.event_number);
exit(1);
}
_OK(dto_event->status, "event status for post_recv");
/* careful when checking cookies:
* Client - receiving multi messages on a single EP
* Server - not receiving on multiple EP's
*/
if (!server || (server && !multi_eps)) {
if (dto_event->transfered_length != msg_size ||
dto_event->user_cookie.as_64 != ctx) {
printf("unexpected event data on recv: len=%d"
" cookie=" F64x " expected %d/%d\n",
(int)dto_event->transfered_length,
dto_event->user_cookie.as_64,
msg_size, ctx);
exit(1);
}
/* Server - receiving one message each across many EP's */
} else {
if (dto_event->transfered_length != msg_size ||
dto_event->user_cookie.as_64 != RECV_BUF_INDEX) {
printf("unexpected event data on recv: len=%d"
"cookie=" F64x " expected %d/%d\n",
(int)dto_event->transfered_length,
dto_event->user_cookie.as_64,
msg_size, RECV_BUF_INDEX);
exit(1);
}
}
/* swap RMR,address info to host order */
if (!server || (server && !multi_eps))
r_iov = (DAT_RMR_TRIPLET *) buf[ctx];
else
r_iov =
(DAT_RMR_TRIPLET *) buf[(i * REG_MEM_COUNT) +
RECV_BUF_INDEX];
if (ud_test)
r_iov = (DAT_RMR_TRIPLET *) ((char *)r_iov + 40);
r_iov->rmr_context = ntoh32(r_iov->rmr_context);
r_iov->virtual_address = ntoh64(r_iov->virtual_address);
r_iov->segment_length = ntoh32(r_iov->segment_length);
printf("Recv RMR message: r_iov(%p):"
" r_key_ctx=%x,va=" F64x ",len=0x%x on EP=%p\n",
r_iov, r_iov->rmr_context,
r_iov->virtual_address,
r_iov->segment_length, dto_event->ep_handle);
}
return (0);
}
/* RC - Client and Server: 1, UD - Client: 1 per EP, Server: 2 per EP's */
void process_conn(int idx)
{
DAT_EVENT event;
DAT_COUNT nmore;
DAT_RETURN status;
int pdata, exp_event;
DAT_IB_EXTENSION_EVENT_DATA *ext_event = (DAT_IB_EXTENSION_EVENT_DATA *)
& event.event_extension_data[0];
DAT_CONNECTION_EVENT_DATA *conn_event =
&event.event_data.connect_event_data;
LOGPRINTF("%s waiting for connect[%d] establishment\n",
server ? "Server" : "Client", idx);
status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
_OK(status, "CONN dat_evd_wait");
LOGPRINTF("%s got connect[%d] event 0x%x, pdata %p sz=%d\n",
server ? "Server" : "Client", idx,
event.event_number, conn_event->private_data,
conn_event->private_data_size);
if (ud_test)
exp_event = DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED;
else
exp_event = DAT_CONNECTION_EVENT_ESTABLISHED;
/* Waiting on CR's or CONN_EST */
if (event.event_number != exp_event ||
(ud_test && event.event_number !=
DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) {
printf("unexpected event, !conn established: 0x%x\n",
event.event_number);
exit(1);
}
/* RC or PASSIVE CONN_EST we are done */
if (!ud_test)
return;
/* store each remote_ah according to remote EP index */
pdata = ntoh32(*((int *)conn_event->private_data));
LOGPRINTF(" Client got private data=0x%x\n", pdata);
/* UD, get AH for sends.
* NOTE: bi-directional AH resolution results in a CONN_EST
* for both outbound connect and inbound CR.
* Use Active CONN_EST which includes server's CR
* pdata for remote_ah idx to send on and ignore PASSIVE CONN_EST.
*
* DAT_IB_UD_PASSIVE_REMOTE_AH == passive side CONN_EST
* DAT_IB_UD_REMOTE_AH == active side CONN_EST
*/
if (ext_event->type == DAT_IB_UD_REMOTE_AH) {
remote_ah[pdata] = ext_event->remote_ah;
printf("remote_ah[%d]: ah=%p, qpn=0x%x "
"addr=%s\n",
pdata, remote_ah[pdata].ah,
remote_ah[pdata].qpn, inet_ntoa(((struct sockaddr_in *)
&remote_ah[pdata].
ia_addr)->sin_addr));
} else if (ext_event->type != DAT_IB_UD_PASSIVE_REMOTE_AH) {
printf("unexpected UD ext_event type: 0x%x\n", ext_event->type);
exit(1);
}
}
void
send_msg(void *data,
DAT_COUNT size,
DAT_LMR_CONTEXT context,
DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags)
{
DAT_LMR_TRIPLET iov;
DAT_EVENT event;
DAT_COUNT nmore;
DAT_RETURN status;
int i, ep_idx = 0, ah_idx = 0;
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
iov.lmr_context = context;
iov.virtual_address = (DAT_VADDR) (uintptr_t) data;
iov.segment_length = (DAT_VLEN) size;
for (i = 0; i < eps; i++) {
if (ud_test) {
/*
* Client and Server: ep[0] and ah[0] on single
* and ep[i] on multiple (-m) endpoint options.
*/
if (multi_eps) {
ep_idx = i;
ah_idx = server ? 0 : i;
}
printf("%s sending on ep=%p to remote_ah: ah=%p"
" qpn=0x%x addr=%s\n",
server ? "Server" : "Client", ep[ep_idx],
remote_ah[ah_idx].ah,
remote_ah[ah_idx].qpn,
inet_ntoa(((struct sockaddr_in *)
&remote_ah[ah_idx].ia_addr)->
sin_addr));
/* client expects all data in on first EP */
status = dat_ib_post_send_ud(ep[ep_idx],
1,
&iov,
&remote_ah[ah_idx],
cookie, flags);
} else {
status = dat_ep_post_send(ep[0], 1, &iov,
cookie, flags);
}
_OK(status, "dat_ep_post_send");
if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
status = dat_evd_wait(dto_evd, DTO_TIMEOUT,
1, &event, &nmore);
_OK(status, "dat_evd_wait after dat_ep_post_send");
if (event.event_number != DAT_DTO_COMPLETION_EVENT &&
ud_test && event.event_number != DAT_IB_DTO_EVENT) {
printf("unexpected event waiting post_send "
"completion - 0x%x\n",
event.event_number);
exit(1);
}
_OK(dto_event->status, "event status for post_send");
}
}
}
int do_fetch_add()
{
DAT_DTO_COOKIE cookie;
DAT_RETURN status;
DAT_EVENT event;
DAT_COUNT nmore;
DAT_LMR_TRIPLET l_iov;
DAT_RMR_TRIPLET r_iov;
volatile DAT_UINT64 *target = (DAT_UINT64 *) buf[RCV_RDMA_BUF_INDEX];
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
DAT_IB_EXTENSION_EVENT_DATA *ext_event =
(DAT_IB_EXTENSION_EVENT_DATA *) & event.event_extension_data[0];
printf("\nDoing FETCH and ADD\n");
/* RMR info already swapped back to host order in connect_ep */
r_iov = *buf[RECV_BUF_INDEX];
l_iov.lmr_context = lmr_atomic_context;
l_iov.virtual_address = (DAT_UINT64) (uintptr_t) atomic_buf;
l_iov.segment_length = BUF_SIZE_ATOMIC;
cookie.as_64 = 0x7777;
if (server) {
/* Wait for Client to finish cmp_swap */
while (*target != 0x98765)
sleep(1);
*target = 0x10;
sleep(1);
status = dat_ib_post_fetch_and_add(ep[0],
(DAT_UINT64) 0x100,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
} else {
/* Wait for Server, no swap so nothing to check */
*target = 0x100;
sleep(1);
status = dat_ib_post_fetch_and_add(ep[0],
(DAT_UINT64) 0x10,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
}
_OK(status, "dat_ib_post_fetch_and_add");
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait for fetch and add");
if (event.event_number != DAT_IB_DTO_EVENT) {
printf("unexpected event after post_fetch_and_add: 0x%x\n",
event.event_number);
exit(1);
}
_OK(dto_event->status, "event status for FETCH and ADD");
if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
printf("unexpected event data of fetch and add : type=%d "
"cookie=%d original%d\n",
(int)ext_event->type,
(int)dto_event->user_cookie.as_64, (int)*atomic_buf);
exit(1);
}
if (server) {
printf("Client original data (on Server) = " F64x ", expected "
"0x100\n", *atomic_buf);
} else {
printf("Server original data (on Client) = " F64x ", expected "
"0x10\n", *atomic_buf);
}
sleep(1);
if (server) {
status = dat_ib_post_fetch_and_add(ep[0],
(DAT_UINT64) 0x100,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
} else {
status = dat_ib_post_fetch_and_add(ep[0],
(DAT_UINT64) 0x10,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
}
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait for second fetch and add");
if (event.event_number != DAT_IB_DTO_EVENT) {
printf("unexpected event after second post_fetch_and_add: "
"0x%x\n", event.event_number);
exit(1);
}
_OK(dto_event->status, "event status for second FETCH and ADD");
if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
printf("unexpected event data of second fetch and add : "
"type=%d cookie=%d original%p\n",
(int)ext_event->type,
(int)dto_event->user_cookie.as_64, atomic_buf);
exit(1);
}
sleep(1); /* wait for other side to complete fetch_add */
if (server) {
printf("Server got original data = " F64x ", expected "
"0x200\n", *atomic_buf);
printf("Client final result (on Server) = " F64x ", expected "
"0x30\n", *target);
if (*atomic_buf != 0x200 || *target != 0x30) {
printf("ERROR: Server FETCH_ADD\n");
exit(1);
}
} else {
printf("Server side original data = " F64x ", expected "
"0x20\n", *atomic_buf);
printf("Server final result (on Client) = " F64x ", expected "
"0x300\n", *target);
if (*atomic_buf != 0x20 || *target != 0x300) {
printf("ERROR: Server FETCH_ADD\n");
exit(1);
}
}
printf("\n FETCH_ADD test - PASSED\n");
return (0);
}
int do_cmp_swap()
{
DAT_DTO_COOKIE cookie;
DAT_RETURN status;
DAT_EVENT event;
DAT_COUNT nmore;
DAT_LMR_TRIPLET l_iov;
DAT_RMR_TRIPLET r_iov;
volatile DAT_UINT64 *target = (DAT_UINT64 *) buf[RCV_RDMA_BUF_INDEX];
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
DAT_IB_EXTENSION_EVENT_DATA *ext_event =
(DAT_IB_EXTENSION_EVENT_DATA *) & event.event_extension_data[0];
printf("\nDoing CMP and SWAP\n");
/* RMR info already swapped back to host order in connect_ep */
r_iov = *buf[RECV_BUF_INDEX];
l_iov.lmr_context = lmr_atomic_context;
l_iov.virtual_address = (DAT_UINT64) (uintptr_t) atomic_buf;
l_iov.segment_length = BUF_SIZE_ATOMIC;
cookie.as_64 = 3333;
if (server) {
*target = 0x12345;
sleep(1);
/* Server does not compare and should not swap */
printf("dtx svr - starting cmp_swap\n");
status = dat_ib_post_cmp_and_swap(ep[0],
(DAT_UINT64) 0x654321,
(DAT_UINT64) 0x6789A,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
printf("dtx svr - done cmp_swap, chk status\n");
} else {
*target = 0x54321;
sleep(1);
printf("dtx cli - starting cmp_swap\n");
/* Client does compare and should swap */
status = dat_ib_post_cmp_and_swap(ep[0],
(DAT_UINT64) 0x12345,
(DAT_UINT64) 0x98765,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
printf("dtx cli - done cmp_swap, chk status\n");
}
_OK(status, "dat_ib_post_cmp_and_swap");
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
_OK(status, "dat_evd_wait for compare and swap");
if (event.event_number != DAT_IB_DTO_EVENT) {
printf("unexpected event after post_cmp_and_swap: 0x%x\n",
event.event_number);
exit(1);
}
_OK(dto_event->status, "event status for CMP and SWAP");
if (ext_event->type != DAT_IB_CMP_AND_SWAP) {
printf("unexpected event data of cmp_swap: type=%d cookie=%d "
"original " F64x "\n",
(int)ext_event->type,
(int)dto_event->user_cookie.as_64, *atomic_buf);
exit(1);
}
sleep(2); /* wait for other side to complete swap */
if (server) {
printf("Server got original data = " F64x ", expected "
"0x54321\n", *atomic_buf);
printf("Client final result (on Server) = " F64x ", expected "
"0x98765\n", *target);
if (*atomic_buf != 0x54321 || *target != 0x98765) {
printf("ERROR: Server CMP_SWAP\n");
exit(1);
}
} else {
printf("Client got original data = " F64x ", expected "
"0x12345\n", *atomic_buf);
printf("Server final result (on Client) = 0x" F64x ", expected "
"0x54321\n", *target);
if (*atomic_buf != 0x12345 || *target != 0x54321) {
printf("ERROR: Client CMP_SWAP\n");
exit(1);
}
}
printf("\n CMP_SWAP test - PASSED\n");
return (0);
}
/* Connect a remote PSP at addr : conn_qual
* return -1 on error */
int psdapl_connect(psdapl_con_info_t *ci, psdapl_info_msg_t *msg)
{
int rc;
DAT_RETURN dat_rc;
rc = psdapl_init_buffers_local(ci);
if (rc) goto err_init_buf;
rc = psdapl_create_ep(ci);
if (rc) goto err_create_ep;
psdapl_init_msg_t res_imsg;
psdapl_init_init_msg(&res_imsg, ci);
dat_rc = dat_ep_connect(ci->ep_handle,
&msg->sock_addr,
msg->conn_qual,
DAT_TIMEOUT_INFINITE /* 5 * 1000 * 1000 */,
sizeof(res_imsg) /* private_data_size */,
&res_imsg /* private_data */,
DAT_QOS_BEST_EFFORT /* DAT_QOS */,
DAT_CONNECT_DEFAULT_FLAG /* DAT_CONNECT_FLAGS */);
if (dat_rc != DAT_SUCCESS) goto err_ep_connect;
DAT_EVENT event;
DAT_COUNT nmore;
event.event_number = -1;
dat_rc = dat_evd_wait(ci->connect_evd_handle,
DAT_TIMEOUT_INFINITE /* 5*1000*1000 timeout in usec*/,
1 /* threshold */,
&event, &nmore);
psdapl_init_msg_t *imsg = NULL;
switch (DAT_GET_TYPE(dat_rc)) {
/*
case DAT_TIMEOUT_EXPIRED:
fprintf(stderr, "<mark (timeout)>\n");
break;
*/
case DAT_SUCCESS:
switch (event.event_number) {
case DAT_CONNECTION_EVENT_TIMED_OUT:
psdapl_dprint(2, "psdapl_connect: event DAT_CONNECTION_EVENT_TIMED_OUT");
break;
case DAT_CONNECTION_EVENT_ESTABLISHED:
psdapl_dprint(3, "psdapl_connect: event DAT_CONNECTION_EVENT_ESTABLISHED");
DAT_CONNECTION_EVENT_DATA *cd = &event.event_data.connect_event_data;
imsg = (psdapl_init_msg_t *)(cd->private_data);
break;
default:
psdapl_dprint(2, "psdapl_connect: unexpected event 0x%x",
(unsigned)event.event_number);
break;
}
break;
default:
psdapl_dprint_dat_err(1, dat_rc, "psdapl_connect: dat_evd_wait()");
break;
}
if (!imsg) goto err_wait;
psdapl_init_buffers_remote(ci, imsg);
return 0;
/* --- */
err_ep_connect:
psdapl_dprint_dat_err(0, dat_rc, "dat_ep_connect() failed");
goto err_all;
/* --- */
err_all:
err_wait:
err_create_ep:
err_init_buf:
/* ToDo: Cleanup recv_evd_handle!! */
/* ToDo: Cleanup connect_evd_handle!! */
/* ToDo: Cleanup bufpairs!!!!! */
return -1;
}
/* Wait for a new connection on the PSP (psdapl_listen()).
*
* return -1 on error */
int psdapl_accept_wait(psdapl_con_info_t *ci)
{
DAT_EVENT event;
DAT_COUNT nmore;
DAT_RETURN dat_rc;
dat_rc = dat_evd_wait(ci->socket->evd_handle,
DAT_TIMEOUT_INFINITE /* 5*1000*1000 timeout in usec*/,
1 /* threshold */,
&event, &nmore);
switch (DAT_GET_TYPE(dat_rc)) {
/*
case DAT_TIMEOUT_EXPIRED:
fprintf(stderr, "<mark (timeout)>\n");
break;
*/
case DAT_SUCCESS:
switch (event.event_number) {
case DAT_CONNECTION_EVENT_TIMED_OUT:
psdapl_dprint(2, "psdapl_accept_wait: event DAT_CONNECTION_EVENT_TIMED_OUT");
break;
case DAT_CONNECTION_REQUEST_EVENT:
psdapl_dprint(3, "psdapl_accept_wait: event DAT_CONNECTION_REQUEST_EVENT");
DAT_CR_ARRIVAL_EVENT_DATA *cr = &event.event_data.cr_arrival_event_data;
DAT_CR_PARAM cr_param;
dat_rc = dat_cr_query(cr->cr_handle, DAT_CR_FIELD_ALL, &cr_param);
assert(dat_rc == DAT_SUCCESS);
psdapl_init_msg_t *imsg = (psdapl_init_msg_t *)(cr_param.private_data);
return _psdapl_get_con_accept(ci, cr->cr_handle, imsg);
break;
/*
DAT_DTO_COMPLETION_EVENT = 0x00001,
DAT_RMR_BIND_COMPLETION_EVENT = 0x01001,
DAT_CONNECTION_REQUEST_EVENT = 0x02001,
DAT_CONNECTION_EVENT_ESTABLISHED = 0x04001,
DAT_CONNECTION_EVENT_PEER_REJECTED = 0x04002,
DAT_CONNECTION_EVENT_NON_PEER_REJECTED = 0x04003,
DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR = 0x04004,
DAT_CONNECTION_EVENT_DISCONNECTED = 0x04005,
DAT_CONNECTION_EVENT_BROKEN = 0x04006,
DAT_CONNECTION_EVENT_TIMED_OUT = 0x04007,
DAT_CONNECTION_EVENT_UNREACHABLE = 0x04008,
DAT_ASYNC_ERROR_EVD_OVERFLOW = 0x08001,
DAT_ASYNC_ERROR_IA_CATASTROPHIC = 0x08002,
DAT_ASYNC_ERROR_EP_BROKEN = 0x08003,
DAT_ASYNC_ERROR_TIMED_OUT = 0x08004,
DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR = 0x08005,
DAT_SOFTWARE_EVENT = 0x10001
*/
default:
psdapl_dprint(2, "psdapl_accept_wait: unexpected event 0x%x",
(unsigned)event.event_number);
break;
}
break;
default:
psdapl_dprint_dat_err(1, dat_rc, "psdapl_accept_wait: dat_evd_wait()");
}
return -1;
}
