Beispiel #1
0
static int fi_ibv_fabric_close(fid_t fid)
{
	struct fi_ibv_fabric *fab;

	fab = container_of(fid, struct fi_ibv_fabric, fabric_fid.fid);
	util_buf_pool_destroy(fab->wce_pool);
	util_buf_pool_destroy(fab->epe_pool);
	free(fid);
	return 0;
}
Beispiel #2
0
static int rxm_ep_txrx_res_open(struct rxm_ep *rxm_ep)
{
	struct rxm_domain *rxm_domain;
	uint8_t local_mr;
	int ret;

	rxm_domain = container_of(rxm_ep->util_ep.domain, struct rxm_domain, util_domain);
	local_mr = rxm_ep->msg_info->mode & FI_LOCAL_MR ? 1 : 0;

	FI_DBG(&rxm_prov, FI_LOG_EP_CTRL, "MSG provider mode & FI_LOCAL_MR: %d\n",
			local_mr);

	ret = rxm_buf_pool_create(local_mr, rxm_ep->msg_info->tx_attr->size,
			sizeof(struct rxm_pkt), &rxm_ep->tx_pool, rxm_domain->msg_domain);
	if (ret)
	        return ret;

	ret = rxm_buf_pool_create(local_mr, rxm_ep->msg_info->rx_attr->size,
			sizeof(struct rxm_rx_buf), &rxm_ep->rx_pool, rxm_domain->msg_domain);
	if (ret)
		goto err1;

	rxm_ep->txe_fs = rxm_txe_fs_create(rxm_ep->rxm_info->tx_attr->size);
	if (!rxm_ep->txe_fs) {
		ret = -FI_ENOMEM;
		goto err2;
	}

	ofi_key_idx_init(&rxm_ep->tx_key_idx, fi_size_bits(rxm_ep->rxm_info->tx_attr->size));

	ret = rxm_recv_queue_init(&rxm_ep->recv_queue, rxm_ep->rxm_info->rx_attr->size);
	if (ret)
		goto err3;

	ret = rxm_recv_queue_init(&rxm_ep->trecv_queue, rxm_ep->rxm_info->rx_attr->size);
	if (ret)
		goto err4;

	return 0;
err4:
	rxm_recv_queue_close(&rxm_ep->recv_queue);
err3:
	rxm_txe_fs_free(rxm_ep->txe_fs);
err2:
	util_buf_pool_destroy(rxm_ep->tx_pool);
err1:
	util_buf_pool_destroy(rxm_ep->rx_pool);
	return ret;
}
Beispiel #3
0
void ofi_mr_cache_cleanup(struct ofi_mr_cache *cache)
{
	struct ofi_mr_entry *entry;
	struct dlist_entry *tmp;

	FI_INFO(cache->domain->prov, FI_LOG_MR, "MR cache stats: "
		"searches %zu, deletes %zu, hits %zu\n",
		cache->search_cnt, cache->delete_cnt, cache->hit_cnt);

	util_mr_cache_process_events(cache);

	dlist_foreach_container_safe(&cache->lru_list, struct ofi_mr_entry,
				     entry, lru_entry, tmp) {
		assert(entry->use_cnt == 0);
		util_mr_uncache_entry(cache, entry);
		dlist_remove_init(&entry->lru_entry);
		util_mr_free_entry(cache, entry);
	}
	cache->mr_storage.destroy(&cache->mr_storage);
	ofi_monitor_del_queue(&cache->nq);
	ofi_atomic_dec32(&cache->domain->ref);
	util_buf_pool_destroy(cache->entry_pool);
	assert(cache->cached_cnt == 0);
	assert(cache->cached_size == 0);
}
Beispiel #4
0
static void rxm_ep_txrx_res_close(struct rxm_ep *rxm_ep)
{
	struct slist_entry *entry;
	struct rxm_rx_buf *rx_buf;

	rxm_recv_queue_close(&rxm_ep->trecv_queue);
	rxm_recv_queue_close(&rxm_ep->recv_queue);

	if (rxm_ep->txe_fs)
		rxm_txe_fs_free(rxm_ep->txe_fs);

	while(!slist_empty(&rxm_ep->rx_buf_list)) {
		entry = slist_remove_head(&rxm_ep->rx_buf_list);
		rx_buf = container_of(entry, struct rxm_rx_buf, entry);
		util_buf_release(rxm_ep->rx_pool, rx_buf);
	}

	util_buf_pool_destroy(rxm_ep->rx_pool);
	util_buf_pool_destroy(rxm_ep->tx_pool);
}
Beispiel #5
0
int fi_ibv_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
		  void *context)
{
	struct fi_ibv_fabric *fab;
	int ret;

	ret = fi_ibv_init_info();
	if (ret)
		return ret;

	ret = fi_ibv_find_fabric(attr);
	if (ret)
		return ret;

	fab = calloc(1, sizeof(*fab));
	if (!fab)
		return -FI_ENOMEM;

	fab->fabric_fid.fid.fclass = FI_CLASS_FABRIC;
	fab->fabric_fid.fid.context = context;
	fab->fabric_fid.fid.ops = &fi_ibv_fi_ops;
	fab->fabric_fid.ops = &fi_ibv_ops_fabric;

	fab->wce_pool = util_buf_pool_create(sizeof(struct fi_ibv_wce), 16, 0, VERBS_WCE_CNT);
	if (!fab->wce_pool) {
		FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC, "Failed to create wce_pool\n");
		ret = -FI_ENOMEM;
		goto err1;
	}

	fab->epe_pool = util_buf_pool_create(sizeof(struct fi_ibv_msg_epe), 16, 0, VERBS_EPE_CNT);
	if (!fab->epe_pool) {
		FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC, "Failed to create epe_pool\n");
		ret = -FI_ENOMEM;
		goto err2;
	}

	*fabric = &fab->fabric_fid;
	return 0;
err2:
	util_buf_pool_destroy(fab->wce_pool);
err1:
	free(fab);
	return ret;
}
Beispiel #6
0
static int rxd_cq_close(struct fid *fid)
{
	int ret;
	struct rxd_cq *cq;

	cq = container_of(fid, struct rxd_cq, util_cq.cq_fid.fid);

	fastlock_acquire(&cq->domain->lock);
	dlist_remove(&cq->dom_entry);
	fastlock_release(&cq->domain->lock);
	fastlock_destroy(&cq->lock);

	ret = fi_close(&cq->dg_cq->fid);
	if (ret)
		return ret;

	ret = ofi_cq_cleanup(&cq->util_cq);
	if (ret)
		return ret;
	util_buf_pool_destroy(cq->unexp_pool);
	free(cq);
	return 0;
}
Beispiel #7
0
int tcpx_progress_close(struct tcpx_progress *progress)
{
	util_buf_pool_destroy(progress->pe_entry_pool);
	return FI_SUCCESS;
}
Beispiel #8
0
static ssize_t mrail_send(struct fid_ep *ep_fid, const void *buf, size_t len,
			  void *desc, fi_addr_t dest_addr, void *context)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_send_common(ep_fid, &iov, &desc, 1, len, dest_addr, 0,
				 context, mrail_comp_flag(ep_fid));
}

static ssize_t mrail_inject(struct fid_ep *ep_fid, const void *buf, size_t len,
			    fi_addr_t dest_addr)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_send_common(ep_fid, &iov, NULL, 1, len, dest_addr, 0,
				 NULL, mrail_inject_flags(ep_fid));
}

static ssize_t mrail_injectdata(struct fid_ep *ep_fid, const void *buf,
				size_t len, uint64_t data, fi_addr_t dest_addr)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_send_common(ep_fid, &iov, NULL, 1, len, dest_addr, data,
				 NULL, (mrail_inject_flags(ep_fid) |
					FI_REMOTE_CQ_DATA));
}

static ssize_t
mrail_tsendmsg(struct fid_ep *ep_fid, const struct fi_msg_tagged *msg,
	       uint64_t flags)
{
	return mrail_tsend_common(ep_fid, msg->msg_iov, msg->desc, msg->iov_count,
				  ofi_total_iov_len(msg->msg_iov, msg->iov_count),
				  msg->addr, msg->tag, msg->data, msg->context,
				  flags | mrail_comp_flag(ep_fid));
}

static ssize_t mrail_tsend(struct fid_ep *ep_fid, const void *buf, size_t len,
			   void *desc, fi_addr_t dest_addr, uint64_t tag,
			   void *context)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_tsend_common(ep_fid, &iov, &desc, 1, len, dest_addr, tag,
				  0, context, mrail_comp_flag(ep_fid));
}

static ssize_t mrail_tsenddata(struct fid_ep *ep_fid, const void *buf, size_t len,
			       void *desc, uint64_t data, fi_addr_t dest_addr,
			       uint64_t tag, void *context)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_tsend_common(ep_fid, &iov, &desc, 1, len, dest_addr, tag,
				  data, context, (mrail_comp_flag(ep_fid) |
						  FI_REMOTE_CQ_DATA));
}

static ssize_t mrail_tinject(struct fid_ep *ep_fid, const void *buf, size_t len,
			     fi_addr_t dest_addr, uint64_t tag)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_tsend_common(ep_fid, &iov, NULL, 1, len, dest_addr, tag,
				  0, NULL, mrail_inject_flags(ep_fid));
}

static ssize_t mrail_tinjectdata(struct fid_ep *ep_fid, const void *buf,
				 size_t len, uint64_t data, fi_addr_t dest_addr,
				 uint64_t tag)
{
	struct iovec iov = {
		.iov_base 	= (void *)buf,
		.iov_len 	= len,
	};
	return mrail_tsend_common(ep_fid, &iov, NULL, 1, len, dest_addr, tag,
				  data, NULL, (mrail_inject_flags(ep_fid) |
					       FI_REMOTE_CQ_DATA));
}

static struct mrail_unexp_msg_entry *
mrail_get_unexp_msg_entry(struct mrail_recv_queue *recv_queue, void *context)
{
	// TODO use buf pool
	// context would be mrail_ep from which u can get the buf pool
	struct mrail_unexp_msg_entry *unexp_msg_entry =
		malloc(sizeof(*unexp_msg_entry) + sizeof(struct fi_cq_tagged_entry));
	return unexp_msg_entry;
}

static int mrail_getname(fid_t fid, void *addr, size_t *addrlen)
{
	struct mrail_ep *mrail_ep =
		container_of(fid, struct mrail_ep, util_ep.ep_fid.fid);
	struct mrail_domain *mrail_domain =
		container_of(mrail_ep->util_ep.domain, struct mrail_domain,
			     util_domain);
	size_t i, offset = 0, rail_addrlen;
	int ret;

	if (*addrlen < mrail_domain->addrlen)
		return -FI_ETOOSMALL;

	for (i = 0; i < mrail_ep->num_eps; i++) {
		rail_addrlen = *addrlen - offset;
		ret = fi_getname(&mrail_ep->rails[i].ep->fid,
				 (char *)addr + offset, &rail_addrlen);
		if (ret) {
			FI_WARN(&mrail_prov, FI_LOG_EP_CTRL,
				"Unable to get name for rail: %zd\n", i);
			return ret;
		}
		offset += rail_addrlen;
	}
	return 0;
}


static void mrail_tx_buf_init(void *pool_ctx, void *buf)
{
	struct mrail_ep *mrail_ep = pool_ctx;
	struct mrail_tx_buf *tx_buf = buf;

	tx_buf->ep		= mrail_ep;
	tx_buf->hdr.version	= MRAIL_HDR_VERSION;
}

static void mrail_ep_free_bufs(struct mrail_ep *mrail_ep)
{
	if (mrail_ep->req_pool)
		util_buf_pool_destroy(mrail_ep->req_pool);

	if (mrail_ep->ooo_recv_pool)
		util_buf_pool_destroy(mrail_ep->ooo_recv_pool);

	if (mrail_ep->tx_buf_pool)
		util_buf_pool_destroy(mrail_ep->tx_buf_pool);

	if (mrail_ep->recv_fs)
		mrail_recv_fs_free(mrail_ep->recv_fs);
}

static int mrail_ep_alloc_bufs(struct mrail_ep *mrail_ep)
{
	struct util_buf_attr attr = {
		.size		= sizeof(struct mrail_tx_buf),
		.alignment	= sizeof(void *),
		.max_cnt	= 0,
		.chunk_cnt	= 64,
		.alloc_hndlr	= NULL,
		.free_hndlr	= NULL,
		.init		= mrail_tx_buf_init,
		.ctx		= mrail_ep,
	};
	size_t buf_size, rxq_total_size = 0;
	struct fi_info *fi;
	int ret;

	for (fi = mrail_ep->info->next; fi; fi = fi->next)
		rxq_total_size += fi->rx_attr->size;

	mrail_ep->recv_fs = mrail_recv_fs_create(rxq_total_size, mrail_init_recv,
						 mrail_ep);
	if (!mrail_ep->recv_fs)
		return -FI_ENOMEM;

	ret = util_buf_pool_create(&mrail_ep->ooo_recv_pool,
				   sizeof(struct mrail_ooo_recv),
				   sizeof(void *), 0, 64);
	if (!mrail_ep->ooo_recv_pool)
		goto err;

	ret = util_buf_pool_create_attr(&attr, &mrail_ep->tx_buf_pool);
	if (!mrail_ep->tx_buf_pool)
		goto err;

	buf_size = (sizeof(struct mrail_req) +
		    (mrail_ep->num_eps * sizeof(struct mrail_subreq)));

	ret = util_buf_pool_create(&mrail_ep->req_pool, buf_size,
				   sizeof(void *), 0, 64);
	if (ret)
		goto err;
	return 0;
err:
	mrail_ep_free_bufs(mrail_ep);
	return ret;
}

static int mrail_ep_close(fid_t fid)
{
	struct mrail_ep *mrail_ep =
		container_of(fid, struct mrail_ep, util_ep.ep_fid.fid);
	int ret, retv = 0;
	size_t i;

	mrail_ep_free_bufs(mrail_ep);

	for (i = 0; i < mrail_ep->num_eps; i++) {
		ret = fi_close(&mrail_ep->rails[i].ep->fid);
		if (ret)
			retv = ret;
	}
	free(mrail_ep->rails);

	ret = ofi_endpoint_close(&mrail_ep->util_ep);
	if (ret)
		retv = ret;
	free(mrail_ep);
	return retv;
}

static int mrail_ep_bind(struct fid *ep_fid, struct fid *bfid, uint64_t flags)
{
	struct mrail_ep *mrail_ep =
		container_of(ep_fid, struct mrail_ep, util_ep.ep_fid.fid);
	struct mrail_cq *mrail_cq;
	struct mrail_av *mrail_av;
	struct util_cntr *cntr;
	int ret = 0;
	size_t i;

	switch (bfid->fclass) {
	case FI_CLASS_AV:
		mrail_av = container_of(bfid, struct mrail_av,
					util_av.av_fid.fid);
		ret = ofi_ep_bind_av(&mrail_ep->util_ep, &mrail_av->util_av);
		if (ret)
			return ret;
		for (i = 0; i < mrail_ep->num_eps; i++) {
			ret = fi_ep_bind(mrail_ep->rails[i].ep,
					 &mrail_av->avs[i]->fid, flags);
			if (ret)
				return ret;
		}
		break;
	case FI_CLASS_CQ:
		mrail_cq = container_of(bfid, struct mrail_cq,
					util_cq.cq_fid.fid);

		ret = ofi_ep_bind_cq(&mrail_ep->util_ep, &mrail_cq->util_cq,
				     flags);
		if (ret)
			return ret;
		for (i = 0; i < mrail_ep->num_eps; i++) {
			ret = fi_ep_bind(mrail_ep->rails[i].ep,
					 &mrail_cq->cqs[i]->fid, flags);
			if (ret)
				return ret;
		}
		break;
	case FI_CLASS_CNTR:
		cntr = container_of(bfid, struct util_cntr, cntr_fid.fid);

		ret = ofi_ep_bind_cntr(&mrail_ep->util_ep, cntr, flags);
		if (ret)
			return ret;
		break;
	case FI_CLASS_EQ:
		ret = -FI_ENOSYS;
		break;
	default:
		FI_WARN(&mrail_prov, FI_LOG_EP_CTRL, "invalid fid class\n");
		ret = -FI_EINVAL;
		break;
	}
	return ret;
}

static int mrail_ep_ctrl(struct fid *fid, int command, void *arg)
{
	struct mrail_ep *mrail_ep;
	size_t i, buf_recv_min = sizeof(struct mrail_hdr);
	int ret;

	mrail_ep = container_of(fid, struct mrail_ep, util_ep.ep_fid.fid);

	switch (command) {
	case FI_ENABLE:
		if (!mrail_ep->util_ep.rx_cq || !mrail_ep->util_ep.tx_cq)
			return -FI_ENOCQ;
		if (!mrail_ep->util_ep.av)
			return -FI_ENOAV;
		for (i = 0; i < mrail_ep->num_eps; i++) {
			ret = fi_setopt(&mrail_ep->rails[i].ep->fid,
					FI_OPT_ENDPOINT, FI_OPT_BUFFERED_MIN,
					&buf_recv_min, sizeof(buf_recv_min));
			if (ret)
				return ret;

			ret = fi_enable(mrail_ep->rails[i].ep);
			if (ret)
				return ret;
		}
		break;
	default:
		return -FI_ENOSYS;
	}
	return 0;
}

static struct fi_ops mrail_ep_fi_ops = {
	.size = sizeof(struct fi_ops),
	.close = mrail_ep_close,
	.bind = mrail_ep_bind,
	.control = mrail_ep_ctrl,
	.ops_open = fi_no_ops_open,
};

static int mrail_ep_setopt(fid_t fid, int level, int optname,
		const void *optval, size_t optlen)
{
	struct mrail_ep *mrail_ep;
	size_t i;
	int ret = 0;

	mrail_ep = container_of(fid, struct mrail_ep, util_ep.ep_fid.fid);

	for (i = 0; i < mrail_ep->num_eps; i++) {
		ret = fi_setopt(&mrail_ep->rails[i].ep->fid, level, optname,
				optval, optlen);
		if (ret)
			return ret;
	}

	return ret;
}

static struct fi_ops_ep mrail_ops_ep = {
	.size = sizeof(struct fi_ops_ep),
	.cancel = fi_no_cancel,
	.getopt = fi_no_getopt,
	.setopt = mrail_ep_setopt,
	.tx_ctx = fi_no_tx_ctx,
	.rx_ctx = fi_no_rx_ctx,
	.rx_size_left = fi_no_rx_size_left,
	.tx_size_left = fi_no_tx_size_left,
};

static struct fi_ops_cm mrail_ops_cm = {
	.size = sizeof(struct fi_ops_cm),
	.setname = fi_no_setname,
	.getname = mrail_getname,
	.getpeer = fi_no_getpeer,
	.connect = fi_no_connect,
	.listen = fi_no_listen,
	.accept = fi_no_accept,
	.reject = fi_no_reject,
	.shutdown = fi_no_shutdown,
	.join = fi_no_join,
};

static struct fi_ops_msg mrail_ops_msg = {
	.size = sizeof(struct fi_ops_msg),
	.recv = mrail_recv,
	.recvv = fi_no_msg_recvv,
	.recvmsg = mrail_recvmsg,
	.send = mrail_send,
	.sendv = fi_no_msg_sendv,
	.sendmsg = mrail_sendmsg,
	.inject = mrail_inject,
	.senddata = fi_no_msg_senddata,
	.injectdata = mrail_injectdata,
};

struct fi_ops_tagged mrail_ops_tagged = {
	.size = sizeof(struct fi_ops_tagged),
	.recv = mrail_trecv,
	.recvv = fi_no_tagged_recvv,
	.recvmsg = mrail_trecvmsg,
	.send = mrail_tsend,
	.sendv = fi_no_tagged_sendv,
	.sendmsg = mrail_tsendmsg,
	.inject = mrail_tinject,
	.senddata = mrail_tsenddata,
	.injectdata = mrail_tinjectdata,
};

void mrail_ep_progress(struct util_ep *ep)
{
	struct mrail_ep *mrail_ep;
	mrail_ep = container_of(ep, struct mrail_ep, util_ep);
	mrail_progress_deferred_reqs(mrail_ep);
}

int mrail_ep_open(struct fid_domain *domain_fid, struct fi_info *info,
		  struct fid_ep **ep_fid, void *context)
{
	struct mrail_domain *mrail_domain =
		container_of(domain_fid, struct mrail_domain,
			     util_domain.domain_fid);
	struct mrail_ep *mrail_ep;
	struct fi_info *fi;
	size_t i;
	int ret;

	if (strcmp(mrail_domain->info->domain_attr->name,
		    info->domain_attr->name)) {
		FI_WARN(&mrail_prov, FI_LOG_EP_CTRL, "info domain name: %s "
			"doesn't match fid_domain name: %s!\n",
			info->domain_attr->name,
			mrail_domain->info->domain_attr->name);
		return -FI_EINVAL;
	}

	mrail_ep = calloc(1, sizeof(*mrail_ep));
	if (!mrail_ep)
		return -FI_ENOMEM;

	// TODO detect changes b/w mrail_domain->info and info arg
	// this may be difficult and we may not support such changes
	mrail_ep->info = mrail_domain->info;
	mrail_ep->num_eps = mrail_domain->num_domains;

	ret = ofi_endpoint_init(domain_fid, &mrail_util_prov, info, &mrail_ep->util_ep,
				context, &mrail_ep_progress);
	if (ret) {
		goto free_ep;
	}

	mrail_ep->rails = calloc(mrail_ep->num_eps, sizeof(*mrail_ep->rails));
	if (!mrail_ep->rails) {
		ret = -FI_ENOMEM;
		goto err;
	}

	for (i = 0, fi = mrail_ep->info->next; fi; fi = fi->next, i++) {
		fi->tx_attr->op_flags &= ~FI_COMPLETION;
		ret = fi_endpoint(mrail_domain->domains[i], fi,
				  &mrail_ep->rails[i].ep, mrail_ep);
		if (ret) {
			FI_WARN(&mrail_prov, FI_LOG_EP_CTRL,
				"Unable to open EP\n");
			goto err;
		}
		mrail_ep->rails[i].info = fi;
	}

	ret = mrail_ep_alloc_bufs(mrail_ep);
	if (ret)
		goto err;

	slist_init(&mrail_ep->deferred_reqs);

	if (mrail_ep->info->caps & FI_DIRECTED_RECV) {
		mrail_recv_queue_init(&mrail_prov, &mrail_ep->recv_queue,
				      mrail_match_recv_addr,
				      mrail_match_unexp_addr,
				      mrail_get_unexp_msg_entry);
		mrail_recv_queue_init(&mrail_prov, &mrail_ep->trecv_queue,
				      mrail_match_recv_addr_tag,
				      mrail_match_unexp_addr_tag,
				      mrail_get_unexp_msg_entry);
	} else {
		mrail_recv_queue_init(&mrail_prov, &mrail_ep->recv_queue,
				      mrail_match_recv_any,
				      mrail_match_unexp_any,
				      mrail_get_unexp_msg_entry);
		mrail_recv_queue_init(&mrail_prov, &mrail_ep->trecv_queue,
				      mrail_match_recv_tag,
				      mrail_match_unexp_tag,
				      mrail_get_unexp_msg_entry);
	}

	ofi_atomic_initialize32(&mrail_ep->tx_rail, 0);
	ofi_atomic_initialize32(&mrail_ep->rx_rail, 0);

	*ep_fid = &mrail_ep->util_ep.ep_fid;
	(*ep_fid)->fid.ops = &mrail_ep_fi_ops;
	(*ep_fid)->ops = &mrail_ops_ep;
	(*ep_fid)->cm = &mrail_ops_cm;
	(*ep_fid)->msg = &mrail_ops_msg;
	(*ep_fid)->tagged = &mrail_ops_tagged;
	(*ep_fid)->rma = &mrail_ops_rma;

	return 0;
err:
	mrail_ep_close(&mrail_ep->util_ep.ep_fid.fid);
free_ep:
	free(mrail_ep);
	return ret;
}