void psmx2_am_ack_rma(struct psmx2_am_request *req)
{
psm2_amarg_t args[8];
if ((req->op & PSMX2_AM_OP_MASK) != PSMX2_AM_REQ_WRITE_LONG)
return;
args[0].u32w0 = PSMX2_AM_REP_WRITE | PSMX2_AM_EOM;
args[0].u32w1 = req->error;
args[1].u64 = (uint64_t)(uintptr_t)req->write.context;
psm2_am_request_short(req->write.peer_addr,
PSMX2_AM_RMA_HANDLER, args, 2, NULL, 0,
PSM2_AM_FLAG_NOREPLY, NULL, NULL);
}
ssize_t psmx2_write_generic(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int nargs;
int am_flags = PSM2_AM_FLAG_ASYNC;
int chunk_size;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
size_t idx;
void *psm2_context;
int no_event;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_write(ep, buf, len, desc, dest_addr,
addr, key, context, flags,
data);
if (!buf)
return -FI_EINVAL;
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(dest_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->tx, dest_addr);
} else if (av && av->type == FI_AV_TABLE) {
idx = dest_addr;
if ((err = psmx2_av_check_table_idx(av, ep_priv->tx, idx)))
return err;
psm2_epaddr = av->tables[ep_priv->tx->id].epaddrs[idx];
} else {
if (!dest_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->tx->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_WRITE, ep_priv,
(void *)buf, len, desc, addr,
key, context, flags, data);
no_event = (flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
req = psmx2_am_request_alloc(ep_priv->tx);
if (!req)
return -FI_ENOMEM;
if (flags & FI_INJECT) {
if (len > psmx2_env.inject_size) {
psmx2_am_request_free(ep_priv->tx, req);
return -FI_EMSGSIZE;
}
req->tmpbuf = malloc(len);
if (!req->tmpbuf) {
psmx2_am_request_free(ep_priv->tx, req);
return -FI_ENOMEM;
}
memcpy(req->tmpbuf, (void *)buf, len);
buf = req->tmpbuf;
} else {
PSMX2_CTXT_TYPE(&req->fi_context) = no_event ?
PSMX2_NOCOMP_WRITE_CONTEXT :
PSMX2_WRITE_CONTEXT;
}
req->no_event = no_event;
req->op = PSMX2_AM_REQ_WRITE;
req->write.buf = (void *)buf;
req->write.len = len;
req->write.addr = addr; /* needed? */
req->write.key = key; /* needed? */
req->write.context = context;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_RMA;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
chunk_size = ep_priv->tx->psm2_am_param.max_request_short;
args[0].u32w0 = 0;
if (psmx2_env.tagged_rma && len > chunk_size) {
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, 0, PSMX2_RMA_TYPE_WRITE);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE_LONG);
args[0].u32w1 = len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr;
args[3].u64 = key;
nargs = 4;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA);
args[4].u64 = data;
nargs++;
}
if (flags & FI_DELIVERY_COMPLETE) {
args[0].u32w0 |= PSMX2_AM_FORCE_ACK;
psm2_context = NULL;
} else {
psm2_context = (void *)&req->fi_context;
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, NULL, 0, am_flags, NULL, NULL);
psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len, psm2_context, &psm2_req);
return 0;
}
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE);
nargs = 4;
while (len > chunk_size) {
args[0].u32w1 = chunk_size;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, (void *)buf, chunk_size, am_flags,
NULL, NULL);
buf = (const uint8_t *)buf + chunk_size;
addr += chunk_size;
len -= chunk_size;
}
args[0].u32w1 = len;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA | PSMX2_AM_EOM);
args[4].u64 = data;
nargs++;
} else {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args, nargs,
(void *)buf, len, am_flags, NULL, NULL);
return 0;
}
ssize_t psmx2_readv_generic(struct fid_ep *ep, const struct iovec *iov,
void *desc, size_t count, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int chunk_size;
size_t offset = 0;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
size_t idx;
size_t total_len, long_len = 0, short_len;
void *long_buf = NULL;
int i;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_readv(ep, iov, desc, count, src_addr,
addr, key, context, flags);
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->tx, src_addr);
} else if (av && av->type == FI_AV_TABLE) {
idx = src_addr;
if ((err = psmx2_av_check_table_idx(av, ep_priv->tx, idx)))
return err;
psm2_epaddr = av->tables[ep_priv->tx->id].epaddrs[idx];
} else {
if (!src_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->tx->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_READV, ep_priv,
(void *)iov, count, desc, addr,
key, context, flags, 0);
total_len = 0;
for (i=0; i<count; i++)
total_len += iov[i].iov_len;
req = psmx2_am_request_alloc(ep_priv->tx);
if (!req)
return -FI_ENOMEM;
req->tmpbuf = malloc(count * sizeof(struct iovec));
if (!req->tmpbuf) {
psmx2_am_request_free(ep_priv->tx, req);
return -FI_ENOMEM;
}
req->iov = req->tmpbuf;
memcpy(req->iov, iov, count * sizeof(struct iovec));
req->op = PSMX2_AM_REQ_READV;
req->read.iov_count = count;
req->read.len = total_len;
req->read.addr = addr; /* needed? */
req->read.key = key; /* needed? */
req->read.context = context;
req->ep = ep_priv;
req->cq_flags = FI_READ | FI_RMA;
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_READ_CONTEXT;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
if (ep_priv->send_selective_completion && !(flags & FI_COMPLETION)) {
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_NOCOMP_READ_CONTEXT;
req->no_event = 1;
}
chunk_size = ep_priv->tx->psm2_am_param.max_reply_short;
if (psmx2_env.tagged_rma) {
for (i=count-1; i>=0; i--) {
if (iov[i].iov_len > chunk_size) {
long_buf = iov[i].iov_base;
long_len = iov[i].iov_len;
break;
} else if (iov[i].iov_len) {
break;
}
}
}
short_len = total_len - long_len;
/* Use short protocol for all but the last segment (long_len) */
args[0].u32w0 = 0;
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ);
args[1].u64 = (uint64_t)(uintptr_t)req;
args[3].u64 = key;
while (short_len > chunk_size) {
args[0].u32w1 = chunk_size;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
addr += chunk_size;
short_len -= chunk_size;
offset += chunk_size;
}
if (!long_len)
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
args[0].u32w1 = short_len;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
/* Use the long protocol for the last segment */
if (long_len) {
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, 0, PSMX2_RMA_TYPE_READ);
PSMX2_SET_MASK(psm2_tagsel, PSMX2_MATCH_ALL, PSMX2_RMA_TYPE_MASK);
psm2_mq_irecv2(ep_priv->tx->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0,
long_buf, long_len,
(void *)&req->fi_context, &psm2_req);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ_LONG);
args[0].u32w1 = long_len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr + short_len;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 4, NULL, 0, 0, NULL, NULL);
}
return 0;
}
ssize_t psmx2_read_generic(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int chunk_size;
size_t offset = 0;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_read(ep, buf, len, desc, src_addr,
addr, key, context, flags);
if (!buf)
return -FI_EINVAL;
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->tx, src_addr);
} else if (av && av->type == FI_AV_TABLE) {
idx = src_addr;
if ((err = psmx2_av_check_table_idx(av, ep_priv->tx, idx)))
return err;
psm2_epaddr = av->tables[ep_priv->tx->id].epaddrs[idx];
} else {
if (!src_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->tx->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_READ, ep_priv,
buf, len, desc, addr, key,
context, flags, 0);
req = psmx2_am_request_alloc(ep_priv->tx);
if (!req)
return -FI_ENOMEM;
req->op = PSMX2_AM_REQ_READ;
req->read.buf = buf;
req->read.len = len;
req->read.addr = addr; /* needed? */
req->read.key = key; /* needed? */
req->read.context = context;
req->ep = ep_priv;
req->cq_flags = FI_READ | FI_RMA;
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_READ_CONTEXT;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
if (ep_priv->send_selective_completion && !(flags & FI_COMPLETION)) {
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_NOCOMP_READ_CONTEXT;
req->no_event = 1;
}
chunk_size = ep_priv->tx->psm2_am_param.max_reply_short;
args[0].u32w0 = 0;
if (psmx2_env.tagged_rma && len > chunk_size) {
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, 0, PSMX2_RMA_TYPE_READ);
PSMX2_SET_MASK(psm2_tagsel, PSMX2_MATCH_ALL, PSMX2_RMA_TYPE_MASK);
psm2_mq_irecv2(ep_priv->tx->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)&req->fi_context, &psm2_req);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ_LONG);
args[0].u32w1 = len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 4, NULL, 0, 0, NULL, NULL);
return 0;
}
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ);
args[1].u64 = (uint64_t)(uintptr_t)req;
args[3].u64 = key;
while (len > chunk_size) {
args[0].u32w1 = chunk_size;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
addr += chunk_size;
len -= chunk_size;
offset += chunk_size;
}
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
args[0].u32w1 = len;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
return 0;
}
ssize_t psmx2_writev_generic(struct fid_ep *ep, const struct iovec *iov,
void **desc, size_t count, fi_addr_t dest_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int nargs;
int am_flags = PSM2_AM_FLAG_ASYNC;
int chunk_size;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
size_t idx;
void *psm2_context;
int no_event;
size_t total_len, len, len_sent;
uint8_t *buf, *p;
int i;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_writev(ep, iov, desc, count,
dest_addr, addr, key,
context, flags, data);
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(dest_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->tx, dest_addr);
} else if (av && av->type == FI_AV_TABLE) {
idx = dest_addr;
if ((err = psmx2_av_check_table_idx(av, ep_priv->tx, idx)))
return err;
psm2_epaddr = av->tables[ep_priv->tx->id].epaddrs[idx];
} else {
if (!dest_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->tx->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_WRITEV, ep_priv,
(void *)iov, count, desc, addr,
key, context, flags, data);
no_event = (flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
total_len = 0;
for (i=0; i<count; i++)
total_len += iov[i].iov_len;
chunk_size = ep_priv->tx->psm2_am_param.max_request_short;
req = psmx2_am_request_alloc(ep_priv->tx);
if (!req)
return -FI_ENOMEM;
/* Case 1: fit into a AM message, then pack and send */
if (total_len <= chunk_size) {
req->tmpbuf = malloc(total_len);
if (!req->tmpbuf) {
psmx2_am_request_free(ep_priv->tx, req);
return -FI_ENOMEM;
}
p = req->tmpbuf;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
memcpy(p, iov[i].iov_base, iov[i].iov_len);
p += iov[i].iov_len;
}
}
buf = req->tmpbuf;
len = total_len;
req->no_event = no_event;
req->op = PSMX2_AM_REQ_WRITE;
req->write.buf = (void *)buf;
req->write.len = len;
req->write.addr = addr; /* needed? */
req->write.key = key; /* needed? */
req->write.context = context;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_RMA;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
args[0].u32w0 = 0;
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE);
args[0].u32w1 = len;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
nargs = 4;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA | PSMX2_AM_EOM);
args[4].u64 = data;
nargs++;
} else {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args, nargs,
(void *)buf, len, am_flags, NULL, NULL);
return 0;
}
if (flags & FI_INJECT) {
psmx2_am_request_free(ep_priv->tx, req);
return -FI_EMSGSIZE;
}
PSMX2_CTXT_TYPE(&req->fi_context) = no_event ?
PSMX2_NOCOMP_WRITE_CONTEXT :
PSMX2_WRITE_CONTEXT;
req->no_event = no_event;
req->op = PSMX2_AM_REQ_WRITE;
req->write.buf = (void *)iov[0].iov_base;
req->write.len = total_len;
req->write.addr = addr; /* needed? */
req->write.key = key; /* needed? */
req->write.context = context;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_RMA;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
/* Case 2: send iov in sequence */
args[0].u32w0 = 0;
len_sent = 0;
for (i=0; i<count; i++) {
if (!iov[i].iov_len)
continue;
/* Case 2.1: use long protocol for the last segment if it is large */
if (psmx2_env.tagged_rma && iov[i].iov_len > chunk_size &&
len_sent + iov[i].iov_len == total_len) {
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, 0, PSMX2_RMA_TYPE_WRITE);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE_LONG);
args[0].u32w1 = iov[i].iov_len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr;
args[3].u64 = key;
nargs = 4;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA);
args[4].u64 = data;
nargs++;
}
if (flags & FI_DELIVERY_COMPLETE) {
args[0].u32w0 |= PSMX2_AM_FORCE_ACK;
psm2_context = NULL;
} else {
psm2_context = (void *)&req->fi_context;
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, NULL, 0, am_flags, NULL, NULL);
psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr, 0,
&psm2_tag, iov[i].iov_base, iov[i].iov_len,
psm2_context, &psm2_req);
return 0;
}
/* Case 2.2: use short protocol all other segments */
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE);
nargs = 4;
buf = iov[i].iov_base;
len = iov[i].iov_len;
while (len > chunk_size) {
args[0].u32w1 = chunk_size;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, (void *)buf, chunk_size, am_flags,
NULL, NULL);
buf += chunk_size;
addr += chunk_size;
len -= chunk_size;
len_sent += chunk_size;
}
args[0].u32w1 = len;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
if (len_sent + len == total_len) {
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA | PSMX2_AM_EOM);
args[4].u64 = data;
nargs++;
} else {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
}
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args, nargs,
(void *)buf, len, am_flags, NULL, NULL);
addr += len;
len_sent += len;
}
return 0;
}
ssize_t psmx2_write_generic(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int nargs;
int am_flags = PSM2_AM_FLAG_ASYNC;
int chunk_size;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
size_t idx;
void *psm2_context;
int no_event;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_WRITE;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->write.ep = ep;
trigger->write.buf = buf;
trigger->write.len = len;
trigger->write.desc = desc;
trigger->write.dest_addr = dest_addr;
trigger->write.addr = addr;
trigger->write.key = key;
trigger->write.context = context;
trigger->write.flags = flags & ~FI_TRIGGER;
trigger->write.data = data;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (!buf)
return -FI_EINVAL;
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
if (!dest_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
vlane = PSMX2_ADDR_TO_VL(dest_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->domain->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_WRITE, ep_priv,
ep_priv->domain->eps[vlane],
(void *)buf, len, desc, addr,
key, context, flags, data);
no_event = (flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
if (flags & FI_INJECT) {
if (len > PSMX2_INJECT_SIZE)
return -FI_EMSGSIZE;
req = malloc(sizeof(*req) + len);
if (!req)
return -FI_ENOMEM;
memset(req, 0, sizeof(*req));
memcpy((uint8_t *)req + sizeof(*req), (void *)buf, len);
buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
PSMX2_CTXT_TYPE(&req->fi_context) = no_event ?
PSMX2_NOCOMP_WRITE_CONTEXT :
PSMX2_WRITE_CONTEXT;
}
req->no_event = no_event;
req->op = PSMX2_AM_REQ_WRITE;
req->write.buf = (void *)buf;
req->write.len = len;
req->write.addr = addr; /* needed? */
req->write.key = key; /* needed? */
req->write.context = context;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_RMA;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
chunk_size = psmx2_am_param.max_request_short;
args[0].u32w0 = 0;
PSMX2_AM_SET_SRC(args[0].u32w0, ep_priv->vlane);
PSMX2_AM_SET_DST(args[0].u32w0, vlane);
if (psmx2_env.tagged_rma && len > chunk_size) {
tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, tag32);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE_LONG);
args[0].u32w1 = len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr;
args[3].u64 = key;
nargs = 4;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA);
args[4].u64 = data;
nargs++;
}
if (flags & FI_DELIVERY_COMPLETE) {
args[0].u32w0 |= PSMX2_AM_FORCE_ACK;
psm2_context = NULL;
} else {
psm2_context = (void *)&req->fi_context;
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, NULL, 0, am_flags, NULL, NULL);
psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len, psm2_context, &psm2_req);
return 0;
}
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE);
nargs = 4;
while (len > chunk_size) {
args[0].u32w1 = chunk_size;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, (void *)buf, chunk_size, am_flags,
NULL, NULL);
buf = (const uint8_t *)buf + chunk_size;
addr += chunk_size;
len -= chunk_size;
}
args[0].u32w1 = len;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA | PSMX2_AM_EOM);
args[4].u64 = data;
nargs++;
} else {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args, nargs,
(void *)buf, len, am_flags, NULL, NULL);
return 0;
}
ssize_t psmx2_readv_generic(struct fid_ep *ep, const struct iovec *iov,
void *desc, size_t count, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int chunk_size;
size_t offset = 0;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32;
size_t idx;
size_t total_len, long_len, short_len;
void *long_buf;
int i;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_READV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->readv.ep = ep;
trigger->readv.iov = iov;
trigger->readv.count = count;
trigger->readv.desc = desc;
trigger->readv.src_addr = src_addr;
trigger->readv.addr = addr;
trigger->readv.key = key;
trigger->readv.context = context;
trigger->readv.flags = flags & ~FI_TRIGGER;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
if (!src_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->domain->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_READV, ep_priv,
ep_priv->domain->eps[vlane],
(void *)iov, count, desc, addr,
key, context, flags, 0);
total_len = 0;
for (i=0; i<count; i++)
total_len += iov[i].iov_len;
req = calloc(1, sizeof(*req) + count * sizeof(struct iovec));
if (!req)
return -FI_ENOMEM;
memcpy(req->iov, iov, count * sizeof(struct iovec));
req->op = PSMX2_AM_REQ_READV;
req->read.iov_count = count;
req->read.len = total_len;
req->read.addr = addr; /* needed? */
req->read.key = key; /* needed? */
req->read.context = context;
req->ep = ep_priv;
req->cq_flags = FI_READ | FI_RMA;
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_READ_CONTEXT;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
if (ep_priv->send_selective_completion && !(flags & FI_COMPLETION)) {
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_NOCOMP_READ_CONTEXT;
req->no_event = 1;
}
chunk_size = psmx2_am_param.max_reply_short;
long_len = 0;
if (psmx2_env.tagged_rma) {
for (i=count-1; i>=0; i--) {
if (iov[i].iov_len > chunk_size) {
long_buf = iov[i].iov_base;
long_len = iov[i].iov_len;
break;
} else if (iov[i].iov_len) {
break;
}
}
}
short_len = total_len - long_len;
/* Use short protocol for all but the last segment (long_len) */
args[0].u32w0 = 0;
PSMX2_AM_SET_SRC(args[0].u32w0, ep_priv->vlane);
PSMX2_AM_SET_DST(args[0].u32w0, vlane);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ);
args[1].u64 = (uint64_t)(uintptr_t)req;
args[3].u64 = key;
while (short_len > chunk_size) {
args[0].u32w1 = chunk_size;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
addr += chunk_size;
short_len -= chunk_size;
offset += chunk_size;
}
if (!long_len)
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
args[0].u32w1 = short_len;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
/* Use the long protocol for the last segment */
if (long_len) {
tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, vlane, ep_priv->vlane);
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, tag32);
PSMX2_SET_TAG(psm2_tagsel, -1ULL, -1);
psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0,
long_buf, long_len,
(void *)&req->fi_context, &psm2_req);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ_LONG);
args[0].u32w1 = long_len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr + short_len;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 4, NULL, 0, 0, NULL, NULL);
}
return 0;
}
ssize_t psmx2_read_generic(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int chunk_size;
size_t offset = 0;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32;
size_t idx;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_READ;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->read.ep = ep;
trigger->read.buf = buf;
trigger->read.len = len;
trigger->read.desc = desc;
trigger->read.src_addr = src_addr;
trigger->read.addr = addr;
trigger->read.key = key;
trigger->read.context = context;
trigger->read.flags = flags & ~FI_TRIGGER;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (!buf)
return -FI_EINVAL;
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
if (!src_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->domain->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_READ, ep_priv,
ep_priv->domain->eps[vlane],
buf, len, desc, addr, key,
context, flags, 0);
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
req->op = PSMX2_AM_REQ_READ;
req->read.buf = buf;
req->read.len = len;
req->read.addr = addr; /* needed? */
req->read.key = key; /* needed? */
req->read.context = context;
req->ep = ep_priv;
req->cq_flags = FI_READ | FI_RMA;
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_READ_CONTEXT;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
if (ep_priv->send_selective_completion && !(flags & FI_COMPLETION)) {
PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_NOCOMP_READ_CONTEXT;
req->no_event = 1;
}
chunk_size = psmx2_am_param.max_reply_short;
args[0].u32w0 = 0;
PSMX2_AM_SET_SRC(args[0].u32w0, ep_priv->vlane);
PSMX2_AM_SET_DST(args[0].u32w0, vlane);
if (psmx2_env.tagged_rma && len > chunk_size) {
tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, vlane, ep_priv->vlane);
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, tag32);
PSMX2_SET_TAG(psm2_tagsel, -1ULL, -1);
psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)&req->fi_context, &psm2_req);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ_LONG);
args[0].u32w1 = len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 4, NULL, 0, 0, NULL, NULL);
return 0;
}
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ);
args[1].u64 = (uint64_t)(uintptr_t)req;
args[3].u64 = key;
while (len > chunk_size) {
args[0].u32w1 = chunk_size;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
addr += chunk_size;
len -= chunk_size;
offset += chunk_size;
}
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
args[0].u32w1 = len;
args[2].u64 = addr;
args[4].u64 = offset;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
args, 5, NULL, 0, 0, NULL, NULL);
return 0;
}
ssize_t psmx2_writev_generic(struct fid_ep *ep, const struct iovec *iov,
void **desc, size_t count, fi_addr_t dest_addr,
uint64_t addr, uint64_t key, void *context,
uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_epaddr_context *epaddr_context;
struct psmx2_am_request *req;
psm2_amarg_t args[8];
int nargs;
int am_flags = PSM2_AM_FLAG_ASYNC;
int chunk_size;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
size_t idx;
void *psm2_context;
int no_event;
size_t total_len, len, len_sent;
uint8_t *buf, *p;
int i;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_WRITEV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->writev.ep = ep;
trigger->writev.iov = iov;
trigger->writev.count = count;
trigger->writev.desc = desc;
trigger->writev.dest_addr = dest_addr;
trigger->writev.addr = addr;
trigger->writev.key = key;
trigger->writev.context = context;
trigger->writev.flags = flags & ~FI_TRIGGER;
trigger->writev.data = data;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
if (!dest_addr)
return -FI_EINVAL;
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
vlane = PSMX2_ADDR_TO_VL(dest_addr);
}
epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
if (epaddr_context->epid == ep_priv->domain->psm2_epid)
return psmx2_rma_self(PSMX2_AM_REQ_WRITEV, ep_priv,
ep_priv->domain->eps[vlane],
(void *)iov, count, desc, addr,
key, context, flags, data);
no_event = (flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
total_len = 0;
for (i=0; i<count; i++)
total_len += iov[i].iov_len;
chunk_size = psmx2_am_param.max_request_short;
/* Case 1: fit into a AM message, then pack and send */
if (total_len <= chunk_size) {
req = malloc(sizeof(*req) + total_len);
if (!req)
return -FI_ENOMEM;
memset(req, 0, sizeof(*req));
p = (uint8_t *)req + sizeof(*req);
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
memcpy(p, iov[i].iov_base, iov[i].iov_len);
p += iov[i].iov_len;
}
}
buf = (uint8_t *)req + sizeof(*req);
len = total_len;
req->no_event = no_event;
req->op = PSMX2_AM_REQ_WRITE;
req->write.buf = (void *)buf;
req->write.len = len;
req->write.addr = addr; /* needed? */
req->write.key = key; /* needed? */
req->write.context = context;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_RMA;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
args[0].u32w0 = 0;
PSMX2_AM_SET_SRC(args[0].u32w0, ep_priv->vlane);
PSMX2_AM_SET_DST(args[0].u32w0, vlane);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE);
args[0].u32w1 = len;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
nargs = 4;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA | PSMX2_AM_EOM);
args[4].u64 = data;
nargs++;
} else {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args, nargs,
(void *)buf, len, am_flags, NULL, NULL);
return 0;
}
if (flags & FI_INJECT)
return -FI_EMSGSIZE;
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
PSMX2_CTXT_TYPE(&req->fi_context) = no_event ?
PSMX2_NOCOMP_WRITE_CONTEXT :
PSMX2_WRITE_CONTEXT;
req->no_event = no_event;
req->op = PSMX2_AM_REQ_WRITE;
req->write.buf = (void *)iov[0].iov_base;
req->write.len = total_len;
req->write.addr = addr; /* needed? */
req->write.key = key; /* needed? */
req->write.context = context;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_RMA;
PSMX2_CTXT_USER(&req->fi_context) = context;
PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
/* Case 2: send iov in sequence */
args[0].u32w0 = 0;
PSMX2_AM_SET_SRC(args[0].u32w0, ep_priv->vlane);
PSMX2_AM_SET_DST(args[0].u32w0, vlane);
len_sent = 0;
for (i=0; i<count; i++) {
if (!iov[i].iov_len)
continue;
/* Case 2.1: use long protocol for the last segment if it is large */
if (psmx2_env.tagged_rma && iov[i].iov_len > chunk_size &&
len_sent + iov[i].iov_len == total_len) {
tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, (uint64_t)req, tag32);
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE_LONG);
args[0].u32w1 = iov[i].iov_len;
args[1].u64 = (uint64_t)req;
args[2].u64 = addr;
args[3].u64 = key;
nargs = 4;
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA);
args[4].u64 = data;
nargs++;
}
if (flags & FI_DELIVERY_COMPLETE) {
args[0].u32w0 |= PSMX2_AM_FORCE_ACK;
psm2_context = NULL;
} else {
psm2_context = (void *)&req->fi_context;
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, NULL, 0, am_flags, NULL, NULL);
psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, 0,
&psm2_tag, iov[i].iov_base, iov[i].iov_len,
psm2_context, &psm2_req);
return 0;
}
/* Case 2.2: use short protocol all other segments */
PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_WRITE);
nargs = 4;
buf = iov[i].iov_base;
len = iov[i].iov_len;
while (len > chunk_size) {
args[0].u32w1 = chunk_size;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, (void *)buf, chunk_size, am_flags,
NULL, NULL);
buf += chunk_size;
addr += chunk_size;
len -= chunk_size;
len_sent += chunk_size;
}
args[0].u32w1 = len;
args[1].u64 = (uint64_t)(uintptr_t)req;
args[2].u64 = addr;
args[3].u64 = key;
if (len_sent + len == total_len) {
if (flags & FI_REMOTE_CQ_DATA) {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_DATA | PSMX2_AM_EOM);
args[4].u64 = data;
nargs++;
} else {
PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
}
}
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args, nargs,
(void *)buf, len, am_flags, NULL, NULL);
addr += len;
len_sent += len;
}
return 0;
}