Esempio n. 1
0
static inline ucs_status_t uct_ugni_post_rdma(uct_ugni_rdma_iface_t *iface,
                                              uct_ugni_ep_t *ep,
                                              uct_ugni_base_desc_t *rdma)
{
    gni_return_t ugni_rc;

    if (ucs_unlikely(!uct_ugni_can_send(ep))) {
        ucs_mpool_put(rdma);
        return UCS_ERR_NO_RESOURCE;
    }

    ugni_rc = GNI_PostRdma(ep->ep, &rdma->desc);
    if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) {
        ucs_mpool_put(rdma);
        if(GNI_RC_ERROR_RESOURCE == ugni_rc || GNI_RC_ERROR_NOMEM == ugni_rc) {
            ucs_debug("GNI_PostRdma failed, Error status: %s %d",
                      gni_err_str[ugni_rc], ugni_rc);
            return UCS_ERR_NO_RESOURCE;
        } else {
            ucs_error("GNI_PostRdma failed, Error status: %s %d",
                      gni_err_str[ugni_rc], ugni_rc);
            return UCS_ERR_IO_ERROR;
        }
    }

    ++ep->outstanding;
    ++iface->super.outstanding;

    return UCS_INPROGRESS;
}
Esempio n. 2
0
int _gnix_rma_post_req(void *data)
{
	struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
	struct gnix_fid_ep *ep = fab_req->gnix_ep;
	struct gnix_nic *nic = ep->nic;
	struct gnix_fid_mem_desc *loc_md;
	struct gnix_tx_descriptor *txd;
	gni_mem_handle_t mdh;
	gni_return_t status;
	int rc;
	int rdma = !!(fab_req->flags & GNIX_RMA_RDMA);
	int indirect = !!(fab_req->flags & GNIX_RMA_INDIRECT);
	int chained = !!(fab_req->flags & GNIX_RMA_CHAINED);
	int inject_err = _gnix_req_inject_err(fab_req);

	rc = _gnix_nic_tx_alloc(nic, &txd);
	if (rc) {
		GNIX_INFO(FI_LOG_EP_DATA,
				"_gnix_nic_tx_alloc() failed: %d\n",
				rc);
		return -FI_ENOSPC;
	}

	txd->completer_fn = __gnix_rma_txd_complete;
	txd->req = fab_req;

	if (rdma) {
		_gnix_convert_key_to_mhdl(
				(gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
				&mdh);
	} else {
		/* Mem handle CRC is not validated during FMA operations.  Skip
		 * this costly calculation. */
		_gnix_convert_key_to_mhdl_no_crc(
				(gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
				&mdh);
	}

	txd->gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma);
	txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
	txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */

	if (unlikely(indirect)) {
		__gnix_rma_fill_pd_indirect_get(fab_req, txd);
	} else if (unlikely(chained)) {
		__gnix_rma_fill_pd_chained_get(fab_req, txd, &mdh);
	} else {
		txd->gni_desc.local_addr = (uint64_t)fab_req->rma.loc_addr;
		txd->gni_desc.length = fab_req->rma.len;
		txd->gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr;

		loc_md = (struct gnix_fid_mem_desc *)fab_req->rma.loc_md;
		if (loc_md) {
			txd->gni_desc.local_mem_hndl = loc_md->mem_hndl;
		}
	}

	txd->gni_desc.remote_mem_hndl = mdh;
	txd->gni_desc.rdma_mode = 0; /* check flags */
	txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */

	{
		gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl;
		gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl;
		GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n",
			  txd->gni_desc.local_addr, txd->gni_desc.remote_addr,
			  txd->gni_desc.length);
		GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n",
			  *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1),
			  *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1),
			  fab_req->rma.rem_mr_key);
	}

	fastlock_acquire(&nic->lock);

	if (unlikely(inject_err)) {
		_gnix_nic_txd_err_inject(nic, txd);
		status = GNI_RC_SUCCESS;
	} else if (chained) {
		status = GNI_CtPostFma(fab_req->vc->gni_ep, &txd->gni_desc);
	} else if (rdma) {
		status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->gni_desc);
	} else {
		status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
	}

	fastlock_release(&nic->lock);

	if (status != GNI_RC_SUCCESS) {
		_gnix_nic_tx_free(nic, txd);
		GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
			  gni_err_str[status]);
	}

	return gnixu_to_fi_errno(status);
}
Esempio n. 3
0
int _gnix_rma_post_req(void *data)
{
	struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
	struct gnix_fid_ep *ep = fab_req->gnix_ep;
	struct gnix_nic *nic = ep->nic;
	struct gnix_fid_mem_desc *loc_md;
	struct gnix_tx_descriptor *txd;
	gni_mem_handle_t mdh;
	gni_return_t status;
	int rc;
	int rdma = !!(fab_req->flags & GNIX_RMA_RDMA);

	rc = _gnix_nic_tx_alloc(nic, &txd);
	if (rc) {
		GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
			 rc);
		return -FI_EAGAIN;
	}

	txd->desc.completer_fn = __gnix_rma_txd_complete;
	txd->desc.req = fab_req;

	_gnix_convert_key_to_mhdl((gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
				  &mdh);
	loc_md = (struct gnix_fid_mem_desc *)fab_req->loc_md;

	//txd->desc.gni_desc.post_id = (uint64_t)fab_req; /* unused */
	txd->desc.gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma);
	txd->desc.gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
	txd->desc.gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
	txd->desc.gni_desc.local_addr = (uint64_t)fab_req->loc_addr;
	if (loc_md) {
		txd->desc.gni_desc.local_mem_hndl = loc_md->mem_hndl;
	}
	txd->desc.gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr;
	txd->desc.gni_desc.remote_mem_hndl = mdh;
	txd->desc.gni_desc.length = fab_req->len;
	txd->desc.gni_desc.rdma_mode = 0; /* check flags */
	txd->desc.gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */

	{
		gni_mem_handle_t *tl_mdh = &txd->desc.gni_desc.local_mem_hndl;
		gni_mem_handle_t *tr_mdh = &txd->desc.gni_desc.remote_mem_hndl;
		GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n",
			  txd->desc.gni_desc.local_addr, txd->desc.gni_desc.remote_addr,
			  txd->desc.gni_desc.length);
		GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n",
			  *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1),
			  *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1),
			  fab_req->rma.rem_mr_key);
	}

	fastlock_acquire(&nic->lock);

	if (rdma) {
		status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->desc.gni_desc);
	} else {
		status = GNI_PostFma(fab_req->vc->gni_ep, &txd->desc.gni_desc);
	}

	fastlock_release(&nic->lock);

	if (status != GNI_RC_SUCCESS) {
		_gnix_nic_tx_free(nic, txd);
		GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
			  gni_err_str[status]);
	}

	return gnixu_to_fi_errno(status);
}
Esempio n. 4
0
int _gnix_rma_post_rdma_chain_req(void *data)
{
	struct gnix_fab_req *req = (struct gnix_fab_req *)data;
	struct gnix_fid_ep *ep = req->gnix_ep;
	struct gnix_nic *nic = ep->nic;
	struct gnix_tx_descriptor *bte_txd, *ct_txd;
	gni_mem_handle_t mdh;
	gni_return_t status;
	int rc;
	int inject_err = _gnix_req_inject_err(req);
	int head_off, head_len, tail_len;
	int fma_chain = 0;

	rc = _gnix_nic_tx_alloc(nic, &bte_txd);
	if (rc) {
		GNIX_INFO(FI_LOG_EP_DATA,
			  "BTE _gnix_nic_tx_alloc() failed: %d\n",
			  rc);
		return -FI_ENOSPC;
	}

	rc = _gnix_nic_tx_alloc(nic, &ct_txd);
	if (rc) {
		_gnix_nic_tx_free(nic, bte_txd);
		GNIX_INFO(FI_LOG_EP_DATA,
			  "CT _gnix_nic_tx_alloc() failed: %d\n",
			  rc);
		return -FI_ENOSPC;
	}

	_gnix_convert_key_to_mhdl(
			(gnix_mr_key_t *)&req->rma.rem_mr_key,
			&mdh);

	/* BTE TXD */
	bte_txd->completer_fn = __gnix_rma_txd_complete;
	bte_txd->req = req;
	bte_txd->gni_desc.type = GNI_POST_RDMA_GET;
	bte_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
	bte_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */

	head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK;
	head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
	tail_len = (req->rma.rem_addr + req->rma.len) & GNI_READ_ALIGN_MASK;

	bte_txd->gni_desc.local_addr = (uint64_t)req->rma.loc_addr + head_len;
	bte_txd->gni_desc.remote_addr = (uint64_t)req->rma.rem_addr + head_len;
	bte_txd->gni_desc.length = req->rma.len - head_len - tail_len;

	bte_txd->gni_desc.remote_mem_hndl = mdh;
	bte_txd->gni_desc.rdma_mode = 0; /* check flags */
	bte_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
	bte_txd->gni_desc.local_mem_hndl = req->rma.loc_md->mem_hndl;

	/* FMA TXD */
	ct_txd->completer_fn = __gnix_rma_txd_complete;
	ct_txd->req = req;
	ct_txd->gni_desc.type = GNI_POST_FMA_GET;
	ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
	ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */

	ct_txd->gni_desc.remote_mem_hndl = mdh;
	ct_txd->gni_desc.rdma_mode = 0; /* check flags */
	ct_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
	ct_txd->gni_desc.local_mem_hndl = nic->int_bufs_mdh;
	ct_txd->gni_desc.length = GNI_READ_ALIGN;

	if (head_off) {
		ct_txd->gni_desc.remote_addr =
				req->rma.rem_addr & ~GNI_READ_ALIGN_MASK;
		ct_txd->gni_desc.local_addr = (uint64_t)ct_txd->int_buf;

		if (tail_len) {
			ct_txd->gni_desc.next_descr = &ct_txd->gni_ct_descs[0];
			ct_txd->gni_ct_descs[0].ep_hndl = req->vc->gni_ep;
			ct_txd->gni_ct_descs[0].length = GNI_READ_ALIGN;
			ct_txd->gni_ct_descs[0].remote_addr =
					(req->rma.rem_addr +
					 req->rma.len) & ~GNI_READ_ALIGN_MASK;
			ct_txd->gni_ct_descs[0].remote_mem_hndl = mdh;
			ct_txd->gni_ct_descs[0].local_addr =
					(uint64_t)ct_txd->int_buf +
					GNI_READ_ALIGN;
			ct_txd->gni_ct_descs[0].local_mem_hndl =
					nic->int_bufs_mdh;
			ct_txd->gni_ct_descs[0].next_descr = NULL;
			fma_chain = 1;
		}
	} else {
		ct_txd->gni_desc.remote_addr =
				(req->rma.rem_addr +
				 req->rma.len) & ~GNI_READ_ALIGN_MASK;
		ct_txd->gni_desc.local_addr =
				(uint64_t)ct_txd->int_buf + GNI_READ_ALIGN;
	}

	fastlock_acquire(&nic->lock);

	if (unlikely(inject_err)) {
		_gnix_nic_txd_err_inject(nic, bte_txd);
		status = GNI_RC_SUCCESS;
	} else {
		status = GNI_PostRdma(req->vc->gni_ep,
				      &bte_txd->gni_desc);
	}

	if (status != GNI_RC_SUCCESS) {
		fastlock_release(&nic->lock);
		_gnix_nic_tx_free(nic, ct_txd);
		_gnix_nic_tx_free(nic, bte_txd);

		GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
			  gni_err_str[status]);
		return gnixu_to_fi_errno(status);
	}

	if (unlikely(inject_err)) {
		_gnix_nic_txd_err_inject(nic, ct_txd);
		status = GNI_RC_SUCCESS;
	} else if (fma_chain) {
		status = GNI_CtPostFma(req->vc->gni_ep,
				       &ct_txd->gni_desc);
	} else {
		status = GNI_PostFma(req->vc->gni_ep,
				     &ct_txd->gni_desc);
	}

	if (status != GNI_RC_SUCCESS) {
		fastlock_release(&nic->lock);
		_gnix_nic_tx_free(nic, ct_txd);

		/* Wait for the first TX to complete, then retransmit the
		 * entire thing. */
		atomic_set(&req->rma.outstanding_txds, 1);
		req->rma.status = GNI_RC_TRANSACTION_ERROR;

		GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
			  gni_err_str[status]);
		return FI_SUCCESS;
	}

	fastlock_release(&nic->lock);

	/* Wait for both TXs to complete, then process the request. */
	atomic_set(&req->rma.outstanding_txds, 2);
	req->rma.status = 0;

	return FI_SUCCESS;
}