/*
* poll a soft CQ
* This will loop over all the hard CQs within, collecting results.
* Since this routine is an inline and is always called with format as
* a constant, I am counting on the compiler optimizing away all the switches
* on format.
*/
static inline ssize_t
usdf_cq_read_common_soft(struct fid_cq *fcq, void *buf, size_t count,
enum fi_cq_format format)
{
struct usdf_cq *cq;
uint8_t *entry;
uint8_t *last;
void *tail;
size_t entry_len;
cq = cq_ftou(fcq);
if (cq->cq_comp.uc_status != 0) {
return -FI_EAVAIL;
}
/* progress... */
usdf_domain_progress(cq->cq_domain);
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
entry_len = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
entry_len = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
entry_len = sizeof(struct fi_cq_data_entry);
break;
default:
USDF_WARN("unexpected CQ format, internal error\n");
return -FI_EOPNOTSUPP;
}
entry = buf;
last = entry + (entry_len * count);
tail = cq->c.soft.cq_tail;
// XXX ... handle error comps
while (entry < last && tail != cq->c.soft.cq_head) {
memcpy(entry, tail, entry_len);
entry += entry_len;
tail = (uint8_t *)tail + entry_len;
if (tail == cq->c.soft.cq_end) {
tail = cq->c.soft.cq_comps;
}
}
cq->c.soft.cq_tail = tail;
if (entry > (uint8_t *)buf) {
return (entry - (uint8_t *)buf) / entry_len;
} else {
return -FI_EAGAIN;
}
}
/*
* poll a hard CQ
* Since this routine is an inline and is always called with format as
* a constant, I am counting on the compiler optimizing away all the switches
* on format.
*/
static inline ssize_t
usdf_cq_read_common(struct fid_cq *fcq, void *buf, size_t count,
enum fi_cq_format format)
{
struct usdf_cq *cq;
uint8_t *entry;
uint8_t *last;
size_t entry_len;
ssize_t ret;
cq = cq_ftou(fcq);
if (cq->cq_comp.uc_status != 0)
return -FI_EAVAIL;
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
entry_len = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
entry_len = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
entry_len = sizeof(struct fi_cq_data_entry);
break;
default:
return 0;
}
ret = 0;
entry = buf;
last = entry + (entry_len * count);
while (entry < last) {
ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp);
if (ret == -EAGAIN)
break;
if (cq->cq_comp.uc_status != 0) {
ret = -FI_EAVAIL;
break;
}
ret = usdf_cq_copy_cq_entry(entry, &cq->cq_comp, format);
if (ret < 0)
return ret;
entry += entry_len;
}
if (entry > (uint8_t *)buf)
return (entry - (uint8_t *)buf) / entry_len;
else
return ret;
}
static ssize_t
usdf_cq_read_data(struct fid_cq *fcq, void *buf, size_t count)
{
struct usdf_cq *cq;
struct fi_cq_data_entry *entry;
struct fi_cq_data_entry *last;
ssize_t ret;
cq = cq_ftou(fcq);
if (cq->cq_comp.uc_status != 0) {
return -FI_EAVAIL;
}
ret = 0;
entry = buf;
last = entry + count;
while (entry < last) {
ret = usd_poll_cq(cq->cq_cq, &cq->cq_comp);
if (ret == -EAGAIN) {
ret = 0;
break;
}
if (cq->cq_comp.uc_status != 0) {
ret = -FI_EAVAIL;
break;
}
entry->op_context = cq->cq_comp.uc_context;
entry->flags = 0;
entry->len = cq->cq_comp.uc_bytes;
entry->buf = 0; /* XXX */
entry->data = 0;
entry++;
}
if (entry > (struct fi_cq_data_entry *)buf) {
return entry - (struct fi_cq_data_entry *)buf;
} else {
return ret;
}
}
static ssize_t
usdf_cq_readfrom_context_soft(struct fid_cq *fcq, void *buf, size_t count,
fi_addr_t *src_addr)
{
struct usdf_cq *cq;
struct usd_cq_impl *ucq;
struct fi_cq_entry *entry;
struct fi_cq_entry *last;
ssize_t ret;
struct cq_desc *cq_desc;
struct usdf_ep *ep;
struct sockaddr_in sin;
struct usd_udp_hdr *hdr;
uint16_t index;
cq = cq_ftou(fcq);
if (cq->cq_comp.uc_status != 0) {
return -FI_EAVAIL;
}
ucq = to_cqi(cq->c.hard.cq_cq);
ret = 0;
entry = buf;
last = entry + count;
while (entry < last) {
cq_desc = (struct cq_desc *)((uint8_t *)ucq->ucq_desc_ring +
(ucq->ucq_next_desc << 4));
ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp);
if (ret == -EAGAIN) {
ret = 0;
break;
}
if (cq->cq_comp.uc_status != 0) {
ret = -FI_EAVAIL;
break;
}
if (cq->cq_comp.uc_type == USD_COMPTYPE_RECV) {
index = le16_to_cpu(cq_desc->completed_index) &
CQ_DESC_COMP_NDX_MASK;
ep = cq->cq_comp.uc_qp->uq_context;
hdr = ep->e.dg.ep_hdr_ptr[index];
memset(&sin, 0, sizeof(sin));
sin.sin_addr.s_addr = hdr->uh_ip.saddr;
sin.sin_port = hdr->uh_udp.source;
ret = fi_av_insert(av_utof(ep->e.dg.ep_av), &sin, 1,
src_addr, 0, NULL);
if (ret != 1) {
*src_addr = FI_ADDR_NOTAVAIL;
}
++src_addr;
}
entry->op_context = cq->cq_comp.uc_context;
entry++;
}
if (entry > (struct fi_cq_entry *)buf) {
return entry - (struct fi_cq_entry *)buf;
} else {
return ret;
}
}
/*
* poll a hard CQ
* Since this routine is an inline and is always called with format as
* a constant, I am counting on the compiler optimizing away all the switches
* on format.
*/
static inline ssize_t
usdf_cq_read_common(struct fid_cq *fcq, void *buf, size_t count,
enum fi_cq_format format)
{
struct usdf_cq *cq;
uint8_t *entry;
uint8_t *last;
size_t entry_len;
struct fi_cq_entry *ctx_entry;
struct fi_cq_msg_entry *msg_entry;
struct fi_cq_data_entry *data_entry;
ssize_t ret;
cq = cq_ftou(fcq);
if (cq->cq_comp.uc_status != 0) {
return -FI_EAVAIL;
}
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
entry_len = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
entry_len = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
entry_len = sizeof(struct fi_cq_data_entry);
break;
default:
return 0;
}
ret = 0;
entry = buf;
last = entry + (entry_len * count);
while (entry < last) {
ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp);
if (ret == -EAGAIN) {
break;
}
if (cq->cq_comp.uc_status != 0) {
ret = -FI_EAVAIL;
break;
}
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
ctx_entry = (struct fi_cq_entry *)entry;
ctx_entry->op_context = cq->cq_comp.uc_context;
break;
case FI_CQ_FORMAT_MSG:
msg_entry = (struct fi_cq_msg_entry *)entry;
msg_entry->op_context = cq->cq_comp.uc_context;
msg_entry->flags = 0;
msg_entry->len = cq->cq_comp.uc_bytes;
break;
case FI_CQ_FORMAT_DATA:
data_entry = (struct fi_cq_data_entry *)entry;
data_entry->op_context = cq->cq_comp.uc_context;
data_entry->flags = 0;
data_entry->len = cq->cq_comp.uc_bytes;
data_entry->buf = 0; /* XXX */
data_entry->data = 0;
break;
default:
return 0;
}
entry += entry_len;
}
if (entry > (uint8_t *)buf) {
return (entry - (uint8_t *)buf) / entry_len;
} else {
return ret;
}
}
/*
* poll a soft CQ
* This will loop over all the hard CQs within, collecting results.
* Since this routine is an inline and is always called with format as
* a constant, I am counting on the compiler optimizing away all the switches
* on format.
*/
static inline ssize_t
usdf_cq_read_common_soft(struct fid_cq *fcq, void *buf, size_t count,
enum fi_cq_format format)
{
struct usdf_cq *cq;
uint8_t *entry;
uint8_t *last;
struct usdf_cq_soft_entry *tail;
size_t entry_len;
ssize_t ret;
cq = cq_ftou(fcq);
if (cq->cq_comp.uc_status != 0) {
return -FI_EAVAIL;
}
/* progress... */
usdf_domain_progress(cq->cq_domain);
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
entry_len = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
entry_len = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
entry_len = sizeof(struct fi_cq_data_entry);
break;
default:
USDF_WARN("unexpected CQ format, internal error\n");
return -FI_EOPNOTSUPP;
}
entry = buf;
last = entry + (entry_len * count);
tail = cq->c.soft.cq_tail;
while (entry < last) {
/* If the head and tail are equal and the last
* operation was a read then that means we have an
* empty queue.
*/
if ((tail == cq->c.soft.cq_head) &&
(cq->c.soft.cq_last_op == USDF_SOFT_CQ_READ))
break;
if (tail->cse_prov_errno > 0) {
if (entry > (uint8_t *) buf)
break;
else
return -FI_EAVAIL;
}
ret = usdf_cq_copy_soft_entry(entry, tail, format);
if (ret < 0) {
return ret;
}
entry += entry_len;
tail++;
if (tail == cq->c.soft.cq_end) {
tail = cq->c.soft.cq_comps;
}
cq->c.soft.cq_last_op = USDF_SOFT_CQ_READ;
}
cq->c.soft.cq_tail = tail;
if (entry > (uint8_t *)buf) {
return (entry - (uint8_t *)buf) / entry_len;
} else {
return -FI_EAGAIN;
}
}
static ssize_t
usdf_cq_sread_common_soft(struct fid_cq *fcq, void *buf, size_t count, const void *cond,
int timeout_ms, enum fi_cq_format format)
{
struct usdf_cq *cq;
uint8_t *entry;
uint8_t *last;
struct usdf_cq_soft_entry *tail;
size_t entry_len;
size_t sleep_time_us;
size_t time_spent_us = 0;
ssize_t ret;
cq = cq_ftou(fcq);
if (cq->cq_attr.wait_obj == FI_WAIT_NONE)
return -FI_EOPNOTSUPP;
sleep_time_us = SREAD_INIT_SLEEP_TIME_US;
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
entry_len = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
entry_len = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
entry_len = sizeof(struct fi_cq_data_entry);
break;
default:
USDF_WARN("unexpected CQ format, internal error\n");
return -FI_EOPNOTSUPP;
}
entry = buf;
last = entry + (entry_len * count);
while (1) {
/* progress... */
usdf_domain_progress(cq->cq_domain);
tail = cq->c.soft.cq_tail;
while (entry < last) {
/* If the head and tail are equal and the last
* operation was a read then that means we have an
* empty queue.
*/
if ((tail == cq->c.soft.cq_head) &&
(cq->c.soft.cq_last_op ==
USDF_SOFT_CQ_READ))
break;
if (tail->cse_prov_errno > 0) {
if (entry > (uint8_t *)buf)
break;
else
return -FI_EAVAIL;
}
ret = usdf_cq_copy_soft_entry(entry, tail, format);
if (ret < 0)
return ret;
entry += entry_len;
tail++;
if (tail == cq->c.soft.cq_end)
tail = cq->c.soft.cq_comps;
cq->c.soft.cq_last_op = USDF_SOFT_CQ_READ;
}
if (entry > (uint8_t *)buf) {
cq->c.soft.cq_tail = tail;
return (entry - (uint8_t *)buf) / entry_len;
} else {
if (timeout_ms >= 0 &&
(time_spent_us >= 1000 * timeout_ms))
break;
usleep(sleep_time_us);
time_spent_us += sleep_time_us;
/* exponentially back off up to a limit */
if (sleep_time_us < SREAD_MAX_SLEEP_TIME_US)
sleep_time_us *= SREAD_EXP_BASE;
sleep_time_us = MIN(sleep_time_us,
SREAD_MAX_SLEEP_TIME_US);
}
}
return -FI_EAGAIN;
}
static inline ssize_t
usdf_cq_sread_common(struct fid_cq *fcq, void *buf, size_t count, const void *cond,
int timeout_ms, enum fi_cq_format format)
{
struct usdf_cq *cq;
uint8_t *entry;
uint8_t *last;
size_t entry_len;
ssize_t ret;
size_t sleep_time_us;
size_t time_spent_us = 0;
sleep_time_us = SREAD_INIT_SLEEP_TIME_US;
cq = cq_ftou(fcq);
if (cq->cq_attr.wait_obj == FI_WAIT_NONE)
return -FI_EOPNOTSUPP;
if (cq->cq_comp.uc_status != 0)
return -FI_EAVAIL;
switch (format) {
case FI_CQ_FORMAT_CONTEXT:
entry_len = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
entry_len = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
entry_len = sizeof(struct fi_cq_data_entry);
break;
default:
return 0;
}
ret = 0;
entry = buf;
last = entry + (entry_len * count);
while (entry < last) {
ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp);
if (ret == -EAGAIN) {
if (entry > (uint8_t *)buf)
break;
if (timeout_ms >= 0 &&
(time_spent_us >= 1000 * timeout_ms))
break;
usleep(sleep_time_us);
time_spent_us += sleep_time_us;
/* exponentially back off up to a limit */
if (sleep_time_us < SREAD_MAX_SLEEP_TIME_US)
sleep_time_us *= SREAD_EXP_BASE;
sleep_time_us = MIN(sleep_time_us,
SREAD_MAX_SLEEP_TIME_US);
continue;
}
if (cq->cq_comp.uc_status != 0) {
if (entry > (uint8_t *) buf)
break;
else
return -FI_EAVAIL;
}
ret = usdf_cq_copy_cq_entry(entry, &cq->cq_comp, format);
if (ret < 0)
return ret;
entry += entry_len;
}
if (entry > (uint8_t *)buf)
return (entry - (uint8_t *)buf) / entry_len;
return -FI_EAGAIN;
}
