static ssize_t psmx_inject(struct fid_ep *ep, const void *buf, size_t len,
fi_addr_t dest_addr, uint64_t addr, uint64_t key)
{
struct psmx_fid_ep *ep_priv;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, NULL, dest_addr, addr, key,
NULL, ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION, 0);
}
static ssize_t psmx_writedata(struct fid_ep *ep, const void *buf, size_t len, void *desc,
uint64_t data, fi_addr_t dest_addr, uint64_t addr,
uint64_t key, void *context)
{
struct psmx_fid_ep *ep_priv;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, desc, dest_addr, addr, key, context,
ep_priv->flags | FI_REMOTE_CQ_DATA, data);
}
static ssize_t psmx_write(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, uint64_t addr,
uint64_t key, void *context)
{
struct psmx_fid_ep *ep_priv;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, desc, dest_addr, addr, key, context,
ep_priv->tx_flags, 0);
}
static ssize_t psmx_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
uint64_t flags)
{
if (!msg || msg->iov_count != 1 || !msg->msg_iov || !msg->rma_iov)
return -FI_EINVAL;
return _psmx_write(ep, msg->msg_iov[0].iov_base,
msg->msg_iov[0].iov_len,
msg->desc ? msg->desc[0] : NULL, msg->addr,
msg->rma_iov[0].addr, msg->rma_iov[0].key,
msg->context, flags, msg->data);
}
int psmx_process_trigger(struct psmx_fid_domain *domain, struct psmx_trigger *trigger)
{
switch (trigger->op) {
case PSMX_TRIGGERED_SEND:
_psmx_send(trigger->send.ep,
trigger->send.buf,
trigger->send.len,
trigger->send.desc,
trigger->send.dest_addr,
trigger->send.context,
trigger->send.flags);
break;
case PSMX_TRIGGERED_RECV:
_psmx_recv(trigger->recv.ep,
trigger->recv.buf,
trigger->recv.len,
trigger->recv.desc,
trigger->recv.src_addr,
trigger->recv.context,
trigger->recv.flags);
break;
case PSMX_TRIGGERED_TSEND:
_psmx_tagged_send(trigger->tsend.ep,
trigger->tsend.buf,
trigger->tsend.len,
trigger->tsend.desc,
trigger->tsend.dest_addr,
trigger->tsend.tag,
trigger->tsend.context,
trigger->tsend.flags);
break;
case PSMX_TRIGGERED_TRECV:
_psmx_tagged_recv(trigger->trecv.ep,
trigger->trecv.buf,
trigger->trecv.len,
trigger->trecv.desc,
trigger->trecv.src_addr,
trigger->trecv.tag,
trigger->trecv.ignore,
trigger->trecv.context,
trigger->trecv.flags);
break;
case PSMX_TRIGGERED_WRITE:
_psmx_write(trigger->write.ep,
trigger->write.buf,
trigger->write.len,
trigger->write.desc,
trigger->write.dest_addr,
trigger->write.addr,
trigger->write.key,
trigger->write.context,
trigger->write.flags,
trigger->write.data);
break;
case PSMX_TRIGGERED_READ:
_psmx_read(trigger->read.ep,
trigger->read.buf,
trigger->read.len,
trigger->read.desc,
trigger->read.src_addr,
trigger->read.addr,
trigger->read.key,
trigger->read.context,
trigger->read.flags);
break;
case PSMX_TRIGGERED_ATOMIC_WRITE:
_psmx_atomic_write(trigger->atomic_write.ep,
trigger->atomic_write.buf,
trigger->atomic_write.count,
trigger->atomic_write.desc,
trigger->atomic_write.dest_addr,
trigger->atomic_write.addr,
trigger->atomic_write.key,
trigger->atomic_write.datatype,
trigger->atomic_write.atomic_op,
trigger->atomic_write.context,
trigger->atomic_write.flags);
break;
case PSMX_TRIGGERED_ATOMIC_READWRITE:
_psmx_atomic_readwrite(trigger->atomic_readwrite.ep,
trigger->atomic_readwrite.buf,
trigger->atomic_readwrite.count,
trigger->atomic_readwrite.desc,
trigger->atomic_readwrite.result,
trigger->atomic_readwrite.result_desc,
trigger->atomic_readwrite.dest_addr,
trigger->atomic_readwrite.addr,
trigger->atomic_readwrite.key,
trigger->atomic_readwrite.datatype,
trigger->atomic_readwrite.atomic_op,
trigger->atomic_readwrite.context,
trigger->atomic_readwrite.flags);
break;
case PSMX_TRIGGERED_ATOMIC_COMPWRITE:
_psmx_atomic_compwrite(trigger->atomic_compwrite.ep,
trigger->atomic_compwrite.buf,
trigger->atomic_compwrite.count,
trigger->atomic_compwrite.desc,
trigger->atomic_compwrite.compare,
trigger->atomic_compwrite.compare_desc,
trigger->atomic_compwrite.result,
trigger->atomic_compwrite.result_desc,
trigger->atomic_compwrite.dest_addr,
trigger->atomic_compwrite.addr,
trigger->atomic_compwrite.key,
trigger->atomic_compwrite.datatype,
trigger->atomic_compwrite.atomic_op,
trigger->atomic_compwrite.context,
trigger->atomic_compwrite.flags);
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CQ,
"%d unsupported op\n", trigger->op);
break;
}
free(trigger);
return 0;
}
void psmx_cntr_check_trigger(struct psmx_fid_cntr *cntr)
{
struct psmx_trigger *trigger;
if (!cntr->trigger)
return;
pthread_mutex_lock(&cntr->trigger_lock);
trigger = cntr->trigger;
while (trigger) {
if (cntr->counter < trigger->threshold)
break;
cntr->trigger = trigger->next;
switch (trigger->op) {
case PSMX_TRIGGERED_SEND:
_psmx_send(trigger->send.ep,
trigger->send.buf,
trigger->send.len,
trigger->send.desc,
trigger->send.dest_addr,
trigger->send.context,
trigger->send.flags);
break;
case PSMX_TRIGGERED_RECV:
_psmx_recv(trigger->recv.ep,
trigger->recv.buf,
trigger->recv.len,
trigger->recv.desc,
trigger->recv.src_addr,
trigger->recv.context,
trigger->recv.flags);
break;
case PSMX_TRIGGERED_TSEND:
_psmx_tagged_send(trigger->tsend.ep,
trigger->tsend.buf,
trigger->tsend.len,
trigger->tsend.desc,
trigger->tsend.dest_addr,
trigger->tsend.tag,
trigger->tsend.context,
trigger->tsend.flags);
break;
case PSMX_TRIGGERED_TRECV:
_psmx_tagged_recv(trigger->trecv.ep,
trigger->trecv.buf,
trigger->trecv.len,
trigger->trecv.desc,
trigger->trecv.src_addr,
trigger->trecv.tag,
trigger->trecv.ignore,
trigger->trecv.context,
trigger->trecv.flags);
break;
case PSMX_TRIGGERED_WRITE:
_psmx_write(trigger->write.ep,
trigger->write.buf,
trigger->write.len,
trigger->write.desc,
trigger->write.dest_addr,
trigger->write.addr,
trigger->write.key,
trigger->write.context,
trigger->write.flags,
trigger->write.data);
break;
case PSMX_TRIGGERED_READ:
_psmx_read(trigger->read.ep,
trigger->read.buf,
trigger->read.len,
trigger->read.desc,
trigger->read.src_addr,
trigger->read.addr,
trigger->read.key,
trigger->read.context,
trigger->read.flags);
break;
case PSMX_TRIGGERED_ATOMIC_WRITE:
_psmx_atomic_write(trigger->atomic_write.ep,
trigger->atomic_write.buf,
trigger->atomic_write.count,
trigger->atomic_write.desc,
trigger->atomic_write.dest_addr,
trigger->atomic_write.addr,
trigger->atomic_write.key,
trigger->atomic_write.datatype,
trigger->atomic_write.atomic_op,
trigger->atomic_write.context,
trigger->atomic_write.flags);
break;
case PSMX_TRIGGERED_ATOMIC_READWRITE:
_psmx_atomic_readwrite(trigger->atomic_readwrite.ep,
trigger->atomic_readwrite.buf,
trigger->atomic_readwrite.count,
trigger->atomic_readwrite.desc,
trigger->atomic_readwrite.result,
trigger->atomic_readwrite.result_desc,
trigger->atomic_readwrite.dest_addr,
trigger->atomic_readwrite.addr,
trigger->atomic_readwrite.key,
trigger->atomic_readwrite.datatype,
trigger->atomic_readwrite.atomic_op,
trigger->atomic_readwrite.context,
trigger->atomic_readwrite.flags);
break;
case PSMX_TRIGGERED_ATOMIC_COMPWRITE:
_psmx_atomic_compwrite(trigger->atomic_compwrite.ep,
trigger->atomic_compwrite.buf,
trigger->atomic_compwrite.count,
trigger->atomic_compwrite.desc,
trigger->atomic_compwrite.compare,
trigger->atomic_compwrite.compare_desc,
trigger->atomic_compwrite.result,
trigger->atomic_compwrite.result_desc,
trigger->atomic_compwrite.dest_addr,
trigger->atomic_compwrite.addr,
trigger->atomic_compwrite.key,
trigger->atomic_compwrite.datatype,
trigger->atomic_compwrite.atomic_op,
trigger->atomic_compwrite.context,
trigger->atomic_compwrite.flags);
break;
default:
PSMX_DEBUG("%s: %d unsupported op\n", __func__, trigger->op);
break;
}
free(trigger);
}
pthread_mutex_unlock(&cntr->trigger_lock);
}