Пример #1
0
/* -----------------------------------------------------------
 * Post a send buffer
 */
DAT_BOOLEAN
DT_post_send_buffer(DT_Tdep_Print_Head * phead,
		    DAT_EP_HANDLE ep_handle, Bpool * bp, int index, int size)
{
	unsigned char *buff = DT_Bpool_GetBuffer(bp, index);
	DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV(bp, index);
	DAT_LMR_CONTEXT lmr_c = DT_Bpool_GetLMR(bp, index);
	DAT_DTO_COOKIE cookie;
	DAT_RETURN ret;

	/*
	 * Prep the inputs
	 */
	iov->virtual_address = (DAT_VADDR) (uintptr_t) buff;
	iov->segment_length = size;
	iov->lmr_context = lmr_c;
	cookie.as_64 = (DAT_UINT64) 0UL;
	cookie.as_ptr = (DAT_PVOID) buff;

	DT_Tdep_PT_Debug(3,
			 (phead, "Post-Send #%d [%p, %x]\n", index, buff,
			  size));

	/* Post the recv buffer */
	ret = dat_ep_post_send(ep_handle,
			       1, iov, cookie, DAT_COMPLETION_DEFAULT_FLAG);
	if (ret != DAT_SUCCESS) {
		DT_Tdep_PT_Printf(phead,
				  "Test Error: dat_ep_post_send failed: %s\n",
				  DT_RetToString(ret));
		DT_Test_Error();
		return false;
	}
	return true;
}
Пример #2
0
void
send_msg(void *data,
	 DAT_COUNT size,
	 DAT_LMR_CONTEXT context,
	 DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags)
{
	DAT_LMR_TRIPLET iov;
	DAT_EVENT event;
	DAT_COUNT nmore;
	DAT_RETURN status;
	int i, ep_idx = 0, ah_idx = 0;
	DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
	    &event.event_data.dto_completion_event_data;

	iov.lmr_context = context;
	iov.virtual_address = (DAT_VADDR) (uintptr_t) data;
	iov.segment_length = (DAT_VLEN) size;

	for (i = 0; i < eps; i++) {
		if (ud_test) {
			/* 
			 * Client and Server: ep[0] and ah[0] on single 
			 * and ep[i] on multiple (-m) endpoint options. 
			 */
			if (multi_eps) {
				ep_idx = i;
				ah_idx = server ? 0 : i;
			}
			printf("%s sending on ep=%p to remote_ah: ah=%p"
			       " qpn=0x%x addr=%s\n",
			       server ? "Server" : "Client", ep[ep_idx],
			       remote_ah[ah_idx].ah,
			       remote_ah[ah_idx].qpn,
			       inet_ntoa(((struct sockaddr_in *)
					  &remote_ah[ah_idx].ia_addr)->
					 sin_addr));

			/* client expects all data in on first EP */
			status = dat_ib_post_send_ud(ep[ep_idx],
						     1,
						     &iov,
						     &remote_ah[ah_idx],
						     cookie, flags);

		} else {
			status = dat_ep_post_send(ep[0], 1, &iov,
						  cookie, flags);
		}
		_OK(status, "dat_ep_post_send");

		if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
			status = dat_evd_wait(dto_evd, DTO_TIMEOUT,
					      1, &event, &nmore);
			_OK(status, "dat_evd_wait after dat_ep_post_send");

			if (event.event_number != DAT_DTO_COMPLETION_EVENT &&
			    ud_test && event.event_number != DAT_IB_DTO_EVENT) {
				printf("unexpected event waiting post_send "
				       "completion - 0x%x\n",
				       event.event_number);
				exit(1);
			}
			_OK(dto_event->status, "event status for post_send");
		}
	}
}
static inline int mca_btl_udapl_sendrecv(mca_btl_udapl_module_t* btl,
        DAT_EP_HANDLE* endpoint)
{
    int rc;
    mca_btl_udapl_frag_t* frag;
    DAT_DTO_COOKIE cookie;
    static int32_t connection_seq = 1;
    uint32_t flags = 0;
    mca_btl_base_endpoint_t* btl_endpoint = NULL; /* endpoint required by
                                                   * mca_btl_udapl_alloc has not
                                                   * been created at this point
                                                   */
    
    /* Post a receive to get the peer's address data */
    frag = (mca_btl_udapl_frag_t*)
        mca_btl_udapl_alloc(
                            &btl->super,
                            btl_endpoint, 
                            MCA_BTL_NO_ORDER,
                            sizeof(mca_btl_udapl_addr_t) +
                            sizeof(int32_t),
                            flags);
    cookie.as_ptr = frag;

    frag->type = MCA_BTL_UDAPL_CONN_RECV;

    rc = dat_ep_post_recv(endpoint, 1,
            &frag->triplet, cookie, DAT_COMPLETION_DEFAULT_FLAG);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_recv",
            major, minor));
        return OMPI_ERROR;
    }


    /* Send our local address data over this EP */
    frag = (mca_btl_udapl_frag_t*)
        mca_btl_udapl_alloc(
                            &btl->super, 
                            btl_endpoint, 
                            MCA_BTL_NO_ORDER,
                            sizeof(mca_btl_udapl_addr_t) +
                            sizeof(int32_t),
                            flags);
    cookie.as_ptr = frag;

    memcpy(frag->segment.seg_addr.pval,
            &btl->udapl_addr, sizeof(mca_btl_udapl_addr_t));
    memcpy((char *)frag->segment.seg_addr.pval + sizeof(mca_btl_udapl_addr_t),
            &connection_seq, sizeof(int32_t));
    connection_seq++;

    frag->type = MCA_BTL_UDAPL_CONN_SEND;

    rc = dat_ep_post_send(endpoint, 1,
            &frag->triplet, cookie, DAT_COMPLETION_DEFAULT_FLAG);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_send",
            major, minor));
        return OMPI_ERROR;
    }

    return OMPI_SUCCESS;
}
Пример #4
0
/* -----------------------------------------------------------
 * Post a send buffer on each of this thread's EPs.
 */
bool
DT_handle_send_op(DT_Tdep_Print_Head * phead,
		  Ep_Context_t * ep_context,
		  unsigned int num_eps, int op_indx, bool poll)
{
	unsigned int i, j;
	unsigned char *completion_reaped;
	unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE];
	bool rc = false;

	if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) {
		completion_reaped = lcomp;
		bzero((void *)completion_reaped,
			sizeof(unsigned char) * num_eps);
	}
	else {
		completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char));
		if (!completion_reaped) {
			return false;
		}
	}

	for (i = 0; i < num_eps; i++) {
		Transaction_Test_Op_t *op = &ep_context[i].op[op_indx];
		DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV(op->bp, 0);
		DAT_DTO_COOKIE cookie;
		DAT_RETURN ret;

		/* Prep the inputs */
		for (j = 0; j < op->num_segs; j++) {
			iov[j].virtual_address = (DAT_VADDR) (uintptr_t)
			    DT_Bpool_GetBuffer(op->bp, j);
			iov[j].segment_length = op->seg_size;
			iov[j].lmr_context = DT_Bpool_GetLMR(op->bp, j);
		}
		cookie.as_64 = ((((DAT_UINT64) i) << 32)
				| (((uintptr_t) DT_Bpool_GetBuffer(op->bp, 0)) &
				   0xffffffffUL));

		/* Post the send */
		ret = dat_ep_post_send(ep_context[i].ep_handle,
				       op->num_segs,
				       iov,
				       cookie, DAT_COMPLETION_DEFAULT_FLAG);

		if (ret != DAT_SUCCESS) {
			DT_Tdep_PT_Printf(phead,
					  "Test Error: dat_ep_post_send failed: %s\n",
					  DT_RetToString(ret));
			DT_Test_Error();
			goto xit;
		}
	}

	for (i = 0; i < num_eps; i++) {
		Transaction_Test_Op_t *op = &ep_context[i].op[op_indx];

		if (op->reap_send_on_recv && !op->server_initiated) {
			/* we will reap the send on the recv (Client SR) */
			rc = true;
			goto xit;
		}
	}

	/* reap the send completion */
	for (i = 0; i < num_eps; i++) {
		Transaction_Test_Op_t *op;
		DAT_DTO_COMPLETION_EVENT_DATA dto_stat;
		DAT_DTO_COOKIE dto_cookie;
		unsigned int epnum;

		if (!DT_dto_event_reap
		    (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) {
			goto xit;
		}

		epnum = dto_stat.user_cookie.as_64 >> 32;
		if (epnum > num_eps) {
			DT_Tdep_PT_Printf(phead,
					  "Test Error: Send: Invalid endpoint completion reaped.\n"
					  "\tEndpoint: 0x%p, Cookie: 0x" F64x
					  ", Length: " F64u "\n",
					  dto_stat.ep_handle,
					  dto_stat.user_cookie.as_64,
					  dto_stat.transfered_length);
			DT_Test_Error();
			goto xit;
		}

		op = &ep_context[epnum].op[op_indx];

		dto_cookie.as_64 = ((((DAT_UINT64) epnum) << 32)
				    |
				    (((uintptr_t) DT_Bpool_GetBuffer(op->bp, 0))
				     & 0xffffffffUL));

		if (!DT_dto_check(phead,
				  &dto_stat,
				  ep_context[epnum].ep_handle,
				  op->num_segs * op->seg_size,
				  dto_cookie, "Send")) {
			goto xit;
		}

		if (completion_reaped[epnum]) {
			DT_Tdep_PT_Printf(phead,
					  "Test Error: Send: Secondary completion seen for endpoint 0x%p (%d)\n",
					  ep_context[epnum].ep_handle, epnum);
			DT_Test_Error();
			goto xit;
		}
		completion_reaped[epnum] = 1;
	}

	for (i = 0; i < num_eps; i++) {
		if (completion_reaped[i] == 0) {
			DT_Tdep_PT_Printf(phead,
					  "Test Error: Send: No completion seen for endpoint 0x%p (#%d)\n",
					  ep_context[i].ep_handle, i);
			DT_Test_Error();
			goto xit;
		}
	}

	rc = true;

xit:
	if (completion_reaped != lcomp)
		DT_Mdep_Free(completion_reaped);
	return rc;
}