/// The InfiniBand completion notification handler.
int poll_completion()
{
const int ne_max = 10;
struct ibv_wc wc[ne_max];
int ne;
int ne_total = 0;
while ((ne = ibv_poll_cq(cq_, ne_max, wc))) {
if (ne < 0)
throw InfinibandException("ibv_poll_cq failed");
ne_total += ne;
for (int i = 0; i < ne; ++i) {
if (wc[i].status != IBV_WC_SUCCESS) {
std::ostringstream s;
s << ibv_wc_status_str(wc[i].status) << " for wr_id "
<< static_cast<int>(wc[i].wr_id);
L_(error) << s.str();
continue;
}
on_completion(wc[i]);
}
}
return ne_total;
}
void ExecImpl::post()
{
if (hosts_.size() == 1 && not hosts_.front()->is_on()) { /* FIXME: handle resource failure for parallel tasks too */
/* If the host running the synchro failed, notice it. This way, the asking
* process can be killed if it runs on that host itself */
state_ = SIMIX_FAILED;
} else if (surf_action_ && surf_action_->get_state() == resource::Action::State::FAILED) {
/* If the host running the synchro didn't fail, then the synchro was canceled */
state_ = SIMIX_CANCELED;
} else if (timeout_detector_ && timeout_detector_->get_state() == resource::Action::State::FINISHED) {
state_ = SIMIX_TIMEOUT;
} else {
state_ = SIMIX_DONE;
}
on_completion(*this);
clean_action();
if (timeout_detector_) {
timeout_detector_->unref();
timeout_detector_ = nullptr;
}
/* If there are simcalls associated with the synchro, then answer them */
if (not simcalls_.empty())
finish();
}
void * poll_cq(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (1) {
if (!paused) {
// rdma_debug("get cq event ...");
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
while (ibv_poll_cq(cq, 1, &wc)) {
// rdma_debug("handle cq ...");
on_completion(&wc);
}
} else {
// rdma_debug("wait signal ...");
pthread_mutex_lock(&mutex);
pthread_cond_wait(&resume_cond, &mutex);
pthread_mutex_unlock(&mutex);
}
}
return NULL;
}
void * poll_cq2(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (1) {
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
while (ibv_poll_cq(cq, 1, &wc))
on_completion(&wc);
}
return NULL;
}
void Unit::erase_after(std::function<bool(std::unique_ptr<UnitAction> &)> func, bool run_completed) {
auto position_it = std::find_if(
std::begin(this->action_stack),
std::end(this->action_stack),
func);
if (position_it != std::end(this->action_stack)) {
auto completed_action = std::move(*position_it);
// erase from the stack
this->action_stack.erase(position_it, std::end(this->action_stack));
// perform any completion actions
if (run_completed) {
completed_action->on_completion();
}
}
}
inline void cfio_rdma_client_wait(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (request_stack_size) {
// rdma_debug("get cq event ...");
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));
// rdma_debug("ibv_ack_cq_events...");
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
while (ibv_poll_cq(cq, 1, &wc)) {
// rdma_debug("handle cq ...");
on_completion(&wc);
}
}
}
void poll_cq(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
int ne;
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));//block by default
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
do {
ne = ibv_poll_cq(cq, 1, &wc);
if(ne < 0){
printf("fail to poll completion from the CQ. ret = %d\n", ne);
return;
}
else if(ne == 0)
continue;
else
on_completion(&wc);
} while (ne == 0);
return;
}
void * poll_cq(void *ctx)
{
void* tmp_ctx;
struct ibv_wc wc;
int num_entries, nument = 1;
tmp_cq = NULL;
while(1) {
if (tmp_cq != NULL) {
while ((num_entries = ibv_poll_cq(tmp_cq, nument, &wc))) {
on_completion(&wc);
}
}
if (ibv_get_cq_event(s_ctx->comp_channel, &tmp_cq, &tmp_ctx)) {
}
ibv_ack_cq_events(tmp_cq, 1);
if (ibv_req_notify_cq(tmp_cq, 0) > 0) {
}
}
return 0;
}
void on_completion_client(struct ibv_wc *wc)
{
on_completion(wc, false);
}
void on_completion_server(struct ibv_wc *wc)
{
on_completion(wc, true);
}
void
rdmaOnCompletionClient(struct ibv_wc *wc)
{
on_completion(wc, false);
}
void
rdmaOnCompletionServer(struct ibv_wc *wc)
{
on_completion(wc, true);
}