/*
* \brief callback handler for new connect requests
*
* \param arg argument passed on the init function
* \param user_st pointer to store the user state
*
* \returns SYS_ERR_OK if the connection is accepted
* errval if connection is rejected
*/
static errval_t dma_svc_connect_cb(void *arg,
void **user_st)
{
errval_t err;
struct user_st *st = calloc(1, sizeof(*st));
if (st == NULL) {
return LIB_ERR_MALLOC_FAIL;
}
XDMA_DEBUG("dma_svc_connect_cb user_st = %p\n", st);
err = dma_mem_mgr_init(&st->mem_mgr, 0x0, (1UL << 48) - 1);
if (err_is_fail(err)) {
free(st);
return err;
}
st->phi = arg;
dma_mem_mgr_set_convert_fn(st->mem_mgr, translate_address, st);
*user_st = st;
return SYS_ERR_OK;
}
static void memcpy_req_cb(errval_t err,
dma_req_id_t id,
void *st)
{
XDMA_DEBUG("memcpy_req_cb %lx, %s\n", id, err_getstring(err));
dma_service_send_done(st, err, id);
}
errval_t dma_service_init(struct xeon_phi *phi)
{
errval_t err;
XDMA_DEBUG("Initializing DMA service\n");
struct waitset *ws = get_default_waitset();
err = xeon_phi_dma_export(phi,
svc_export_cb,
svc_connect_cb,
ws,
IDC_EXPORT_FLAGS_DEFAULT);
if (err_is_fail(err)) {
return err;
}
XDMAV_DEBUG("Waiting for export...\n");
while (svc_state == XPM_SVC_STATE_EXPORTING) {
messages_wait_and_handle_next();
}
if (svc_state == XPM_SVC_STATE_EXPORT_FAIL) {
return FLOUNDER_ERR_BIND;
}
svc_state = XPM_SVC_STATE_NS_REGISTERING;
char buf[50];
#ifdef __k1om__
snprintf(buf, 50, "%s.%u", XEON_PHI_DMA_SERVICE_NAME, 0);
#else
snprintf(buf, 50, "%s.%u", XEON_PHI_DMA_SERVICE_NAME, phi->id);
#endif
XDMA_DEBUG("Registering iref [%u] with name [%s]\n", dma_iref, buf);
err = nameservice_register(buf, dma_iref);
if (err_is_fail(err)) {
svc_state = XPM_SVC_STATE_NS_REGISTER_FAIL;
return err;
}
svc_state = XPM_SVC_STATE_RUNNING;
return SYS_ERR_OK;
}
/**
* \brief deregisters a memory region with the client
*
* \param user_st pointer to stored user state
* \param cap the capability to deregister
*
* \returns SYS_ERR_OK if the memory region was removed
* errval if the memory region removal was rejected
*/
static errval_t dma_svc_removeregion_cb(dma_svc_handle_t svc_handle,
struct capref cap)
{
struct user_st *user_st = dma_service_get_user_state(svc_handle);
XDMA_DEBUG("dma_svc_removeregion_cb user_st = %p\n", user_st);
return dma_mem_deregister(user_st->mem_mgr, cap);
}
/**
* \brief executes a DMA memcpy
*
* \param user_st pointer to stored user state
* \param dst the physical destination address
* \param src the physical source address
* \param bytes size of the transfer in bytes
* \param id returns the DMA request ID of the transfer
*
* \returns SYS_ERR_OK if the memory region was removed
* errval if the memory region removal was rejected
*/
static errval_t dma_svc_memcpy_cb(dma_svc_handle_t svc_handle,
lpaddr_t dst,
lpaddr_t src,
size_t bytes,
dma_req_id_t *id)
{
errval_t err;
struct user_st *st = dma_service_get_user_state(svc_handle);
XDMA_DEBUG("dma_svc_memcpy_cb user_st = %p\n", st);
lpaddr_t dma_dst, dma_src;
err = dma_mem_verify(st->mem_mgr, dst, bytes, &dma_dst);
if (err_is_fail(err)) {
return err;
}
err = dma_mem_verify(st->mem_mgr, src, bytes, &dma_src);
if (err_is_fail(err)) {
return err;
}
XDMA_DEBUG("[%016lx]->[%016lx] of %lu bytes\n", dma_src, dma_dst, bytes);
/* both addresses are valid and have been translated now */
struct dma_device *dev = st->phi->dma;
assert(dev);
struct dma_req_setup setup = {
.type = DMA_REQ_TYPE_MEMCPY,
.done_cb = memcpy_req_cb,
.cb_arg = svc_handle,
.args = {
.memcpy = {
.src = dma_src,
.dst = dma_dst,
.bytes = bytes
}
}
};
return dma_request_memcpy(dev, &setup, id);
}
/**
* \brief Xeon Phi DMA service connect handler
*
* \param st pointer to struct xeon_phi
* \param b Flounder binding
*/
static errval_t svc_connect_cb(void *st,
struct xeon_phi_dma_binding *b)
{
errval_t err;
XDMA_DEBUG("New connection to the DMA service.\n");
b->rx_vtbl = dma_rx_vtbl;
struct xeon_phi *phi = st;
err = xdma_mem_init(b, phi);
if (err_is_fail(err)) {
return err;
}
return SYS_ERR_OK;
}
static void dma_exec_response_tx(void *a)
{
errval_t err;
struct dma_exec_resp_st *st = a;
struct event_closure txcont = MKCONT(dma_exec_response_sent, a);
err = xeon_phi_dma_exec_response__tx(st->b, txcont, st->err, st->id);
if (err_is_fail(err)) {
if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
txcont = MKCONT(dma_exec_response_tx, a);
XDMA_DEBUG("dma_exec_response_tx: register sending...\n");
err = st->b->register_send(st->b, get_default_waitset(), txcont);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not send reply");
}
}
return;
}
}
static void dma_exec_call_rx(struct xeon_phi_dma_binding *_binding,
uint64_t src,
uint64_t dst,
uint64_t length)
{
XDMAV_DEBUG("memcopy request [0x%016lx]->[0x%016lx] of size 0x%lx\n",
src, dst, length);
struct dma_exec_resp_st st;
st.b = _binding;
st.sent = 0x0;
lpaddr_t dma_src = xdma_mem_verify(_binding, src, length);
lpaddr_t dma_dst = xdma_mem_verify(_binding, dst, length);
if (!dma_src || !dma_dst) {
st.err = XEON_PHI_ERR_DMA_MEM_REGISTERED;
st.id = 0;
#ifdef XDEBUG_DMA
if (!dma_src) {
XDMA_DEBUG("Memory range not registered: [0x%016lx] [0x%016lx]\n",
src, src+length);
}
if (!dma_dst) {
XDMA_DEBUG("Memory range not registered: [0x%016lx] [0x%016lx]\n",
dst, dst+length);
}
#endif
dma_exec_response_tx(&st);
return;
}
/*
* DMA transfers from host to host are not supported.
*/
if (dma_src > XEON_PHI_SYSMEM_BASE && dma_dst > XEON_PHI_SYSMEM_BASE) {
st.err = XEON_PHI_ERR_DMA_NOT_SUPPORTED;
st.id = 0;
dma_exec_response_tx(&st);
return;
}
struct dma_req_setup setup = {
.type = XDMA_REQ_TYPE_MEMCPY,
.st = _binding,
.cb = dma_service_send_done,
};
setup.info.mem.src = dma_src;
setup.info.mem.dst = dma_dst;
setup.info.mem.bytes = length;
setup.info.mem.dma_id = &st.id;
struct xeon_phi *phi = xdma_mem_get_phi(_binding);
st.err = dma_do_request(phi, &setup);
dma_exec_response_tx(&st);
/*
* XXX: we must wait until the message has been sent, otherwise we may
* trigger sending the done message when we poll in the main message
* loop. This causes the client library to receive a done message
* of an invalid id.
*/
volatile uint8_t *sent_flag = &st.sent;
while(!(*sent_flag)) {
messages_wait_and_handle_next();
}
}
