/* * poll a soft CQ * This will loop over all the hard CQs within, collecting results. * Since this routine is an inline and is always called with format as * a constant, I am counting on the compiler optimizing away all the switches * on format. */ static inline ssize_t usdf_cq_read_common_soft(struct fid_cq *fcq, void *buf, size_t count, enum fi_cq_format format) { struct usdf_cq *cq; uint8_t *entry; uint8_t *last; void *tail; size_t entry_len; cq = cq_ftou(fcq); if (cq->cq_comp.uc_status != 0) { return -FI_EAVAIL; } /* progress... */ usdf_domain_progress(cq->cq_domain); switch (format) { case FI_CQ_FORMAT_CONTEXT: entry_len = sizeof(struct fi_cq_entry); break; case FI_CQ_FORMAT_MSG: entry_len = sizeof(struct fi_cq_msg_entry); break; case FI_CQ_FORMAT_DATA: entry_len = sizeof(struct fi_cq_data_entry); break; default: USDF_WARN("unexpected CQ format, internal error\n"); return -FI_EOPNOTSUPP; } entry = buf; last = entry + (entry_len * count); tail = cq->c.soft.cq_tail; // XXX ... handle error comps while (entry < last && tail != cq->c.soft.cq_head) { memcpy(entry, tail, entry_len); entry += entry_len; tail = (uint8_t *)tail + entry_len; if (tail == cq->c.soft.cq_end) { tail = cq->c.soft.cq_comps; } } cq->c.soft.cq_tail = tail; if (entry > (uint8_t *)buf) { return (entry - (uint8_t *)buf) / entry_len; } else { return -FI_EAGAIN; } }
/* * poll a hard CQ * Since this routine is an inline and is always called with format as * a constant, I am counting on the compiler optimizing away all the switches * on format. */ static inline ssize_t usdf_cq_read_common(struct fid_cq *fcq, void *buf, size_t count, enum fi_cq_format format) { struct usdf_cq *cq; uint8_t *entry; uint8_t *last; size_t entry_len; ssize_t ret; cq = cq_ftou(fcq); if (cq->cq_comp.uc_status != 0) return -FI_EAVAIL; switch (format) { case FI_CQ_FORMAT_CONTEXT: entry_len = sizeof(struct fi_cq_entry); break; case FI_CQ_FORMAT_MSG: entry_len = sizeof(struct fi_cq_msg_entry); break; case FI_CQ_FORMAT_DATA: entry_len = sizeof(struct fi_cq_data_entry); break; default: return 0; } ret = 0; entry = buf; last = entry + (entry_len * count); while (entry < last) { ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp); if (ret == -EAGAIN) break; if (cq->cq_comp.uc_status != 0) { ret = -FI_EAVAIL; break; } ret = usdf_cq_copy_cq_entry(entry, &cq->cq_comp, format); if (ret < 0) return ret; entry += entry_len; } if (entry > (uint8_t *)buf) return (entry - (uint8_t *)buf) / entry_len; else return ret; }
static ssize_t usdf_cq_read_data(struct fid_cq *fcq, void *buf, size_t count) { struct usdf_cq *cq; struct fi_cq_data_entry *entry; struct fi_cq_data_entry *last; ssize_t ret; cq = cq_ftou(fcq); if (cq->cq_comp.uc_status != 0) { return -FI_EAVAIL; } ret = 0; entry = buf; last = entry + count; while (entry < last) { ret = usd_poll_cq(cq->cq_cq, &cq->cq_comp); if (ret == -EAGAIN) { ret = 0; break; } if (cq->cq_comp.uc_status != 0) { ret = -FI_EAVAIL; break; } entry->op_context = cq->cq_comp.uc_context; entry->flags = 0; entry->len = cq->cq_comp.uc_bytes; entry->buf = 0; /* XXX */ entry->data = 0; entry++; } if (entry > (struct fi_cq_data_entry *)buf) { return entry - (struct fi_cq_data_entry *)buf; } else { return ret; } }
static ssize_t usdf_cq_readfrom_context_soft(struct fid_cq *fcq, void *buf, size_t count, fi_addr_t *src_addr) { struct usdf_cq *cq; struct usd_cq_impl *ucq; struct fi_cq_entry *entry; struct fi_cq_entry *last; ssize_t ret; struct cq_desc *cq_desc; struct usdf_ep *ep; struct sockaddr_in sin; struct usd_udp_hdr *hdr; uint16_t index; cq = cq_ftou(fcq); if (cq->cq_comp.uc_status != 0) { return -FI_EAVAIL; } ucq = to_cqi(cq->c.hard.cq_cq); ret = 0; entry = buf; last = entry + count; while (entry < last) { cq_desc = (struct cq_desc *)((uint8_t *)ucq->ucq_desc_ring + (ucq->ucq_next_desc << 4)); ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp); if (ret == -EAGAIN) { ret = 0; break; } if (cq->cq_comp.uc_status != 0) { ret = -FI_EAVAIL; break; } if (cq->cq_comp.uc_type == USD_COMPTYPE_RECV) { index = le16_to_cpu(cq_desc->completed_index) & CQ_DESC_COMP_NDX_MASK; ep = cq->cq_comp.uc_qp->uq_context; hdr = ep->e.dg.ep_hdr_ptr[index]; memset(&sin, 0, sizeof(sin)); sin.sin_addr.s_addr = hdr->uh_ip.saddr; sin.sin_port = hdr->uh_udp.source; ret = fi_av_insert(av_utof(ep->e.dg.ep_av), &sin, 1, src_addr, 0, NULL); if (ret != 1) { *src_addr = FI_ADDR_NOTAVAIL; } ++src_addr; } entry->op_context = cq->cq_comp.uc_context; entry++; } if (entry > (struct fi_cq_entry *)buf) { return entry - (struct fi_cq_entry *)buf; } else { return ret; } }
/* * poll a hard CQ * Since this routine is an inline and is always called with format as * a constant, I am counting on the compiler optimizing away all the switches * on format. */ static inline ssize_t usdf_cq_read_common(struct fid_cq *fcq, void *buf, size_t count, enum fi_cq_format format) { struct usdf_cq *cq; uint8_t *entry; uint8_t *last; size_t entry_len; struct fi_cq_entry *ctx_entry; struct fi_cq_msg_entry *msg_entry; struct fi_cq_data_entry *data_entry; ssize_t ret; cq = cq_ftou(fcq); if (cq->cq_comp.uc_status != 0) { return -FI_EAVAIL; } switch (format) { case FI_CQ_FORMAT_CONTEXT: entry_len = sizeof(struct fi_cq_entry); break; case FI_CQ_FORMAT_MSG: entry_len = sizeof(struct fi_cq_msg_entry); break; case FI_CQ_FORMAT_DATA: entry_len = sizeof(struct fi_cq_data_entry); break; default: return 0; } ret = 0; entry = buf; last = entry + (entry_len * count); while (entry < last) { ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp); if (ret == -EAGAIN) { break; } if (cq->cq_comp.uc_status != 0) { ret = -FI_EAVAIL; break; } switch (format) { case FI_CQ_FORMAT_CONTEXT: ctx_entry = (struct fi_cq_entry *)entry; ctx_entry->op_context = cq->cq_comp.uc_context; break; case FI_CQ_FORMAT_MSG: msg_entry = (struct fi_cq_msg_entry *)entry; msg_entry->op_context = cq->cq_comp.uc_context; msg_entry->flags = 0; msg_entry->len = cq->cq_comp.uc_bytes; break; case FI_CQ_FORMAT_DATA: data_entry = (struct fi_cq_data_entry *)entry; data_entry->op_context = cq->cq_comp.uc_context; data_entry->flags = 0; data_entry->len = cq->cq_comp.uc_bytes; data_entry->buf = 0; /* XXX */ data_entry->data = 0; break; default: return 0; } entry += entry_len; } if (entry > (uint8_t *)buf) { return (entry - (uint8_t *)buf) / entry_len; } else { return ret; } }
/* * poll a soft CQ * This will loop over all the hard CQs within, collecting results. * Since this routine is an inline and is always called with format as * a constant, I am counting on the compiler optimizing away all the switches * on format. */ static inline ssize_t usdf_cq_read_common_soft(struct fid_cq *fcq, void *buf, size_t count, enum fi_cq_format format) { struct usdf_cq *cq; uint8_t *entry; uint8_t *last; struct usdf_cq_soft_entry *tail; size_t entry_len; ssize_t ret; cq = cq_ftou(fcq); if (cq->cq_comp.uc_status != 0) { return -FI_EAVAIL; } /* progress... */ usdf_domain_progress(cq->cq_domain); switch (format) { case FI_CQ_FORMAT_CONTEXT: entry_len = sizeof(struct fi_cq_entry); break; case FI_CQ_FORMAT_MSG: entry_len = sizeof(struct fi_cq_msg_entry); break; case FI_CQ_FORMAT_DATA: entry_len = sizeof(struct fi_cq_data_entry); break; default: USDF_WARN("unexpected CQ format, internal error\n"); return -FI_EOPNOTSUPP; } entry = buf; last = entry + (entry_len * count); tail = cq->c.soft.cq_tail; while (entry < last) { /* If the head and tail are equal and the last * operation was a read then that means we have an * empty queue. */ if ((tail == cq->c.soft.cq_head) && (cq->c.soft.cq_last_op == USDF_SOFT_CQ_READ)) break; if (tail->cse_prov_errno > 0) { if (entry > (uint8_t *) buf) break; else return -FI_EAVAIL; } ret = usdf_cq_copy_soft_entry(entry, tail, format); if (ret < 0) { return ret; } entry += entry_len; tail++; if (tail == cq->c.soft.cq_end) { tail = cq->c.soft.cq_comps; } cq->c.soft.cq_last_op = USDF_SOFT_CQ_READ; } cq->c.soft.cq_tail = tail; if (entry > (uint8_t *)buf) { return (entry - (uint8_t *)buf) / entry_len; } else { return -FI_EAGAIN; } }
static ssize_t usdf_cq_sread_common_soft(struct fid_cq *fcq, void *buf, size_t count, const void *cond, int timeout_ms, enum fi_cq_format format) { struct usdf_cq *cq; uint8_t *entry; uint8_t *last; struct usdf_cq_soft_entry *tail; size_t entry_len; size_t sleep_time_us; size_t time_spent_us = 0; ssize_t ret; cq = cq_ftou(fcq); if (cq->cq_attr.wait_obj == FI_WAIT_NONE) return -FI_EOPNOTSUPP; sleep_time_us = SREAD_INIT_SLEEP_TIME_US; switch (format) { case FI_CQ_FORMAT_CONTEXT: entry_len = sizeof(struct fi_cq_entry); break; case FI_CQ_FORMAT_MSG: entry_len = sizeof(struct fi_cq_msg_entry); break; case FI_CQ_FORMAT_DATA: entry_len = sizeof(struct fi_cq_data_entry); break; default: USDF_WARN("unexpected CQ format, internal error\n"); return -FI_EOPNOTSUPP; } entry = buf; last = entry + (entry_len * count); while (1) { /* progress... */ usdf_domain_progress(cq->cq_domain); tail = cq->c.soft.cq_tail; while (entry < last) { /* If the head and tail are equal and the last * operation was a read then that means we have an * empty queue. */ if ((tail == cq->c.soft.cq_head) && (cq->c.soft.cq_last_op == USDF_SOFT_CQ_READ)) break; if (tail->cse_prov_errno > 0) { if (entry > (uint8_t *)buf) break; else return -FI_EAVAIL; } ret = usdf_cq_copy_soft_entry(entry, tail, format); if (ret < 0) return ret; entry += entry_len; tail++; if (tail == cq->c.soft.cq_end) tail = cq->c.soft.cq_comps; cq->c.soft.cq_last_op = USDF_SOFT_CQ_READ; } if (entry > (uint8_t *)buf) { cq->c.soft.cq_tail = tail; return (entry - (uint8_t *)buf) / entry_len; } else { if (timeout_ms >= 0 && (time_spent_us >= 1000 * timeout_ms)) break; usleep(sleep_time_us); time_spent_us += sleep_time_us; /* exponentially back off up to a limit */ if (sleep_time_us < SREAD_MAX_SLEEP_TIME_US) sleep_time_us *= SREAD_EXP_BASE; sleep_time_us = MIN(sleep_time_us, SREAD_MAX_SLEEP_TIME_US); } } return -FI_EAGAIN; }
static inline ssize_t usdf_cq_sread_common(struct fid_cq *fcq, void *buf, size_t count, const void *cond, int timeout_ms, enum fi_cq_format format) { struct usdf_cq *cq; uint8_t *entry; uint8_t *last; size_t entry_len; ssize_t ret; size_t sleep_time_us; size_t time_spent_us = 0; sleep_time_us = SREAD_INIT_SLEEP_TIME_US; cq = cq_ftou(fcq); if (cq->cq_attr.wait_obj == FI_WAIT_NONE) return -FI_EOPNOTSUPP; if (cq->cq_comp.uc_status != 0) return -FI_EAVAIL; switch (format) { case FI_CQ_FORMAT_CONTEXT: entry_len = sizeof(struct fi_cq_entry); break; case FI_CQ_FORMAT_MSG: entry_len = sizeof(struct fi_cq_msg_entry); break; case FI_CQ_FORMAT_DATA: entry_len = sizeof(struct fi_cq_data_entry); break; default: return 0; } ret = 0; entry = buf; last = entry + (entry_len * count); while (entry < last) { ret = usd_poll_cq(cq->c.hard.cq_cq, &cq->cq_comp); if (ret == -EAGAIN) { if (entry > (uint8_t *)buf) break; if (timeout_ms >= 0 && (time_spent_us >= 1000 * timeout_ms)) break; usleep(sleep_time_us); time_spent_us += sleep_time_us; /* exponentially back off up to a limit */ if (sleep_time_us < SREAD_MAX_SLEEP_TIME_US) sleep_time_us *= SREAD_EXP_BASE; sleep_time_us = MIN(sleep_time_us, SREAD_MAX_SLEEP_TIME_US); continue; } if (cq->cq_comp.uc_status != 0) { if (entry > (uint8_t *) buf) break; else return -FI_EAVAIL; } ret = usdf_cq_copy_cq_entry(entry, &cq->cq_comp, format); if (ret < 0) return ret; entry += entry_len; } if (entry > (uint8_t *)buf) return (entry - (uint8_t *)buf) / entry_len; return -FI_EAGAIN; }