static int sock_poll_poll(struct fid_poll *pollset, void **context, int count)
{
struct sock_poll *poll;
struct sock_cq *cq;
struct sock_eq *eq;
struct sock_cntr *cntr;
struct sock_fid_list *list_item;
struct dlist_entry *p, *head;
int ret_count = 0;
poll = container_of(pollset, struct sock_poll, poll_fid.fid);
head = &poll->fid_list;
for (p = head->next; p != head && ret_count < count; p = p->next) {
list_item = container_of(p, struct sock_fid_list, entry);
switch (list_item->fid->fclass) {
case FI_CLASS_CQ:
cq = container_of(list_item->fid, struct sock_cq, cq_fid);
sock_cq_progress(cq);
fastlock_acquire(&cq->lock);
if (rbfdused(&cq->cq_rbfd)) {
*context++ = cq->cq_fid.fid.context;
ret_count++;
}
fastlock_release(&cq->lock);
break;
case FI_CLASS_CNTR:
cntr = container_of(list_item->fid, struct sock_cntr, cntr_fid);
sock_cntr_progress(cntr);
fastlock_acquire(&cntr->mut);
if (atomic_get(&cntr->value) >= atomic_get(&cntr->threshold)) {
*context++ = cntr->cntr_fid.fid.context;
ret_count++;
}
fastlock_release(&cntr->mut);
break;
case FI_CLASS_EQ:
eq = container_of(list_item->fid, struct sock_eq, eq);
fastlock_acquire(&eq->lock);
if (!dlistfd_empty(&eq->list)) {
*context++ = eq->eq.fid.context;
ret_count++;
}
fastlock_release(&eq->lock);
break;
default:
break;
}
}
return ret_count;
}
static ssize_t sock_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
fi_addr_t *src_addr, const void *cond, int timeout)
{
int ret = 0;
size_t threshold;
struct sock_cq *sock_cq;
uint64_t start_ms = 0, end_ms = 0;
ssize_t cq_entry_len, avail;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (rbused(&sock_cq->cqerr_rb))
return -FI_EAVAIL;
cq_entry_len = sock_cq->cq_entry_size;
if (sock_cq->attr.wait_cond == FI_CQ_COND_THRESHOLD)
threshold = MIN((uintptr_t) cond, count);
else
threshold = count;
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL) {
if (timeout >= 0) {
start_ms = fi_gettime_ms();
end_ms = start_ms + timeout;
}
do {
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
avail = rbfdused(&sock_cq->cq_rbfd);
if (avail)
ret = sock_cq_rbuf_read(sock_cq, buf,
MIN(threshold, avail / cq_entry_len),
src_addr, cq_entry_len);
fastlock_release(&sock_cq->lock);
if (ret == 0 && timeout >= 0) {
if (fi_gettime_ms() >= end_ms)
return -FI_EAGAIN;
}
} while (ret == 0);
} else {
ret = rbfdwait(&sock_cq->cq_rbfd, timeout);
if (ret > 0) {
fastlock_acquire(&sock_cq->lock);
ret = 0;
avail = rbfdused(&sock_cq->cq_rbfd);
if (avail)
ret = sock_cq_rbuf_read(sock_cq, buf,
MIN(threshold, avail / cq_entry_len),
src_addr, cq_entry_len);
fastlock_release(&sock_cq->lock);
}
}
return (ret == 0 || ret == -FI_ETIMEDOUT) ? -FI_EAGAIN : ret;
}
static ssize_t sock_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct sock_cq *sock_cq;
ssize_t ret;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL)
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
if (rbused(&sock_cq->cqerr_rb) >= sizeof(struct fi_cq_err_entry)) {
rbread(&sock_cq->cqerr_rb, buf, sizeof(*buf));
ret = 1;
} else {
ret = -FI_EAGAIN;
}
fastlock_release(&sock_cq->lock);
return ret;
}
static ssize_t sock_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct sock_cq *sock_cq;
ssize_t ret;
struct fi_cq_err_entry entry;
uint32_t api_version;
size_t err_data_size = 0;
void *err_data = NULL;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL)
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
if (ofi_rbused(&sock_cq->cqerr_rb) >= sizeof(struct fi_cq_err_entry)) {
api_version = sock_cq->domain->fab->fab_fid.api_version;
ofi_rbread(&sock_cq->cqerr_rb, &entry, sizeof(entry));
if ((FI_VERSION_GE(api_version, FI_VERSION(1, 5)))
&& buf->err_data && buf->err_data_size) {
err_data = buf->err_data;
err_data_size = buf->err_data_size;
*buf = entry;
buf->err_data = err_data;
/* Fill provided user's buffer */
buf->err_data_size = MIN(entry.err_data_size, err_data_size);
memcpy(buf->err_data, entry.err_data, buf->err_data_size);
} else {
*buf = entry;
}
ret = 1;
} else {
ret = -FI_EAGAIN;
}
fastlock_release(&sock_cq->lock);
return ret;
}
static ssize_t sock_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
fi_addr_t *src_addr, const void *cond, int timeout)
{
int ret = 0;
size_t threshold;
struct sock_cq *sock_cq;
uint64_t start_ms;
ssize_t cq_entry_len, avail;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (ofi_rbused(&sock_cq->cqerr_rb))
return -FI_EAVAIL;
cq_entry_len = sock_cq->cq_entry_size;
if (sock_cq->attr.wait_cond == FI_CQ_COND_THRESHOLD)
threshold = MIN((uintptr_t) cond, count);
else
threshold = count;
start_ms = (timeout >= 0) ? fi_gettime_ms() : 0;
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL) {
while (1) {
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
avail = ofi_rbfdused(&sock_cq->cq_rbfd);
if (avail) {
ret = sock_cq_rbuf_read(sock_cq, buf,
MIN(threshold, (size_t)(avail / cq_entry_len)),
src_addr, cq_entry_len);
}
fastlock_release(&sock_cq->lock);
if (ret)
return ret;
if (timeout >= 0) {
timeout -= (int) (fi_gettime_ms() - start_ms);
if (timeout <= 0)
return -FI_EAGAIN;
}
if (ofi_atomic_get32(&sock_cq->signaled)) {
ofi_atomic_set32(&sock_cq->signaled, 0);
return -FI_ECANCELED;
}
};
} else {
do {
fastlock_acquire(&sock_cq->lock);
ret = 0;
avail = ofi_rbfdused(&sock_cq->cq_rbfd);
if (avail) {
ret = sock_cq_rbuf_read(sock_cq, buf,
MIN(threshold, (size_t)(avail / cq_entry_len)),
src_addr, cq_entry_len);
} else {
ofi_rbfdreset(&sock_cq->cq_rbfd);
}
fastlock_release(&sock_cq->lock);
if (ret && ret != -FI_EAGAIN)
return ret;
if (timeout >= 0) {
timeout -= (int) (fi_gettime_ms() - start_ms);
if (timeout <= 0)
return -FI_EAGAIN;
}
if (ofi_atomic_get32(&sock_cq->signaled)) {
ofi_atomic_set32(&sock_cq->signaled, 0);
return -FI_ECANCELED;
}
ret = ofi_rbfdwait(&sock_cq->cq_rbfd, timeout);
} while (ret > 0);
}
return (ret == 0 || ret == -FI_ETIMEDOUT) ? -FI_EAGAIN : ret;
}
