static ssize_t psmx2_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags)
{
void *buf;
size_t len;
if (!msg || (msg->iov_count && !msg->msg_iov))
return -FI_EINVAL;
if (msg->iov_count > 1) {
return psmx2_sendv_generic(ep, msg->msg_iov, msg->desc,
msg->iov_count, msg->addr,
msg->context, flags,
msg->data);
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
} else {
buf = NULL;
len = 0;
}
return psmx2_send_generic(ep, buf, len,
msg->desc ? msg->desc[0] : NULL,
msg->addr, msg->context, flags,
msg->data);
}
DIRECT_FN
STATIC ssize_t psmx2_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags)
{
void *buf;
size_t len;
assert(msg);
assert(!msg->iov_count || msg->msg_iov);
assert(msg->iov_count <= PSMX2_IOV_MAX_COUNT);
if (msg->iov_count > 1) {
return psmx2_sendv_generic(ep, msg->msg_iov, msg->desc,
msg->iov_count, msg->addr,
msg->context, flags,
msg->data);
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
} else {
buf = NULL;
len = 0;
}
return psmx2_send_generic(ep, buf, len,
msg->desc ? msg->desc[0] : NULL,
msg->addr, msg->context, flags,
msg->data);
}
static ssize_t psmx2_send(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, void *context)
{
struct psmx2_fid_ep *ep_priv;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_send_generic(ep, buf, len, desc, dest_addr, context,
ep_priv->tx_flags, 0);
}
int psmx2_process_trigger(struct psmx2_fid_domain *domain,
struct psmx2_trigger *trigger)
{
switch (trigger->op) {
case PSMX2_TRIGGERED_SEND:
psmx2_send_generic(trigger->send.ep,
trigger->send.buf,
trigger->send.len,
trigger->send.desc,
trigger->send.dest_addr,
trigger->send.context,
trigger->send.flags,
trigger->send.data);
break;
case PSMX2_TRIGGERED_SENDV:
psmx2_sendv_generic(trigger->sendv.ep,
trigger->sendv.iov,
trigger->sendv.desc,
trigger->sendv.count,
trigger->sendv.dest_addr,
trigger->sendv.context,
trigger->sendv.flags,
trigger->sendv.data);
break;
case PSMX2_TRIGGERED_RECV:
psmx2_recv_generic(trigger->recv.ep,
trigger->recv.buf,
trigger->recv.len,
trigger->recv.desc,
trigger->recv.src_addr,
trigger->recv.context,
trigger->recv.flags);
break;
case PSMX2_TRIGGERED_TSEND:
psmx2_tagged_send_generic(trigger->tsend.ep,
trigger->tsend.buf,
trigger->tsend.len,
trigger->tsend.desc,
trigger->tsend.dest_addr,
trigger->tsend.tag,
trigger->tsend.context,
trigger->tsend.flags,
trigger->tsend.data);
break;
case PSMX2_TRIGGERED_TSENDV:
psmx2_tagged_sendv_generic(trigger->tsendv.ep,
trigger->tsendv.iov,
trigger->tsendv.desc,
trigger->tsendv.count,
trigger->tsendv.dest_addr,
trigger->tsendv.tag,
trigger->tsendv.context,
trigger->tsendv.flags,
trigger->tsendv.data);
break;
case PSMX2_TRIGGERED_TRECV:
psmx2_tagged_recv_generic(trigger->trecv.ep,
trigger->trecv.buf,
trigger->trecv.len,
trigger->trecv.desc,
trigger->trecv.src_addr,
trigger->trecv.tag,
trigger->trecv.ignore,
trigger->trecv.context,
trigger->trecv.flags);
break;
case PSMX2_TRIGGERED_WRITE:
psmx2_write_generic(trigger->write.ep,
trigger->write.buf,
trigger->write.len,
trigger->write.desc,
trigger->write.dest_addr,
trigger->write.addr,
trigger->write.key,
trigger->write.context,
trigger->write.flags,
trigger->write.data);
break;
case PSMX2_TRIGGERED_WRITEV:
psmx2_writev_generic(trigger->writev.ep,
trigger->writev.iov,
trigger->writev.desc,
trigger->writev.count,
trigger->writev.dest_addr,
trigger->writev.addr,
trigger->writev.key,
trigger->writev.context,
trigger->writev.flags,
trigger->writev.data);
break;
case PSMX2_TRIGGERED_READ:
psmx2_read_generic(trigger->read.ep,
trigger->read.buf,
trigger->read.len,
trigger->read.desc,
trigger->read.src_addr,
trigger->read.addr,
trigger->read.key,
trigger->read.context,
trigger->read.flags);
break;
case PSMX2_TRIGGERED_READV:
psmx2_readv_generic(trigger->readv.ep,
trigger->readv.iov,
trigger->readv.desc,
trigger->readv.count,
trigger->readv.src_addr,
trigger->readv.addr,
trigger->readv.key,
trigger->readv.context,
trigger->readv.flags);
break;
case PSMX2_TRIGGERED_ATOMIC_WRITE:
psmx2_atomic_write_generic(
trigger->atomic_write.ep,
trigger->atomic_write.buf,
trigger->atomic_write.count,
trigger->atomic_write.desc,
trigger->atomic_write.dest_addr,
trigger->atomic_write.addr,
trigger->atomic_write.key,
trigger->atomic_write.datatype,
trigger->atomic_write.atomic_op,
trigger->atomic_write.context,
trigger->atomic_write.flags);
break;
case PSMX2_TRIGGERED_ATOMIC_READWRITE:
psmx2_atomic_readwrite_generic(
trigger->atomic_readwrite.ep,
trigger->atomic_readwrite.buf,
trigger->atomic_readwrite.count,
trigger->atomic_readwrite.desc,
trigger->atomic_readwrite.result,
trigger->atomic_readwrite.result_desc,
trigger->atomic_readwrite.dest_addr,
trigger->atomic_readwrite.addr,
trigger->atomic_readwrite.key,
trigger->atomic_readwrite.datatype,
trigger->atomic_readwrite.atomic_op,
trigger->atomic_readwrite.context,
trigger->atomic_readwrite.flags);
break;
case PSMX2_TRIGGERED_ATOMIC_COMPWRITE:
psmx2_atomic_compwrite_generic(
trigger->atomic_compwrite.ep,
trigger->atomic_compwrite.buf,
trigger->atomic_compwrite.count,
trigger->atomic_compwrite.desc,
trigger->atomic_compwrite.compare,
trigger->atomic_compwrite.compare_desc,
trigger->atomic_compwrite.result,
trigger->atomic_compwrite.result_desc,
trigger->atomic_compwrite.dest_addr,
trigger->atomic_compwrite.addr,
trigger->atomic_compwrite.key,
trigger->atomic_compwrite.datatype,
trigger->atomic_compwrite.atomic_op,
trigger->atomic_compwrite.context,
trigger->atomic_compwrite.flags);
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CQ,
"%d unsupported op\n", trigger->op);
break;
}
free(trigger);
return 0;
}
ssize_t psmx2_sendv_generic(struct fid_ep *ep, const struct iovec *iov,
void **desc, size_t count, fi_addr_t dest_addr,
void *context, uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t msg_flags;
struct fi_context * fi_context;
int send_flag = 0;
int err;
int no_completion = 0;
struct psmx2_cq_event *event;
size_t real_count;
size_t len, total_len;
char *p;
uint32_t *q;
int i, j;
struct psmx2_sendv_request *req;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_sendv(ep, iov, desc, count,
dest_addr, context, flags,
data);
total_len = 0;
real_count = 0;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
total_len += iov[i].iov_len;
real_count++;
j = i;
}
}
if (real_count == 1)
return psmx2_send_generic(ep, iov[j].iov_base, iov[j].iov_len,
desc ? desc[j] : NULL, dest_addr,
context, flags, data);
req = malloc(sizeof(*req));
if (!req)
return -FI_ENOMEM;
if (total_len <= PSMX2_IOV_BUF_SIZE) {
req->iov_protocol = PSMX2_IOV_PROTO_PACK;
p = req->buf;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
memcpy(p, iov[i].iov_base, iov[i].iov_len);
p += iov[i].iov_len;
}
}
msg_flags = PSMX2_TYPE_MSG;
len = total_len;
} else {
req->iov_protocol = PSMX2_IOV_PROTO_MULTI;
req->iov_done = 0;
req->iov_info.seq_num = (++ep_priv->iov_seq_num) %
PSMX2_IOV_MAX_SEQ_NUM + 1;
req->iov_info.count = (uint32_t)real_count;
req->iov_info.total_len = (uint32_t)total_len;
q = req->iov_info.len;
for (i=0; i<count; i++) {
if (iov[i].iov_len)
*q++ = (uint32_t)iov[i].iov_len;
}
msg_flags = PSMX2_TYPE_MSG | PSMX2_IOV_BIT;
len = (3 + real_count) * sizeof(uint32_t);
}
av = ep_priv->av;
assert(av);
psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->tx, dest_addr, av->type);
if (flags & FI_REMOTE_CQ_DATA)
msg_flags |= PSMX2_IMM_BIT;
PSMX2_SET_TAG(psm2_tag, 0ULL, data, msg_flags);
if ((flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION)))
no_completion = 1;
if (flags & FI_INJECT) {
if (len > psmx2_env.inject_size) {
free(req);
return -FI_EMSGSIZE;
}
err = psm2_mq_send2(ep_priv->tx->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, req->buf, len);
free(req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr, 0);
if (ep_priv->send_cq && !no_completion) {
event = psmx2_cq_create_event(
ep_priv->send_cq,
context, NULL, flags, len,
(uint64_t) data,
0 /* tag */,
0 /* olen */,
0 /* err */);
if (event)
psmx2_cq_enqueue_event(ep_priv->send_cq, event);
else
return -FI_ENOMEM;
}
return 0;
}
req->no_completion = no_completion;
req->user_context = context;
req->comp_flag = FI_MSG;
fi_context = &req->fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_SENDV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, req->buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK) {
free(req);
return psmx2_errno(err);
}
PSMX2_CTXT_REQ(fi_context) = psm2_req;
if (req->iov_protocol == PSMX2_IOV_PROTO_MULTI) {
fi_context = &req->fi_context_iov;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_SEND_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_SET_TAG(psm2_tag, req->iov_info.seq_num, 0, PSMX2_TYPE_IOV_PAYLOAD);
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
err = psm2_mq_isend2(ep_priv->tx->psm2_mq,
psm2_epaddr, send_flag, &psm2_tag,
iov[i].iov_base, iov[i].iov_len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
}
}
}
return 0;
}