/*! Helper function to allocate the iobuffer needed by an eventq
* - it ensures the eventq has the correct alignment for the NIC
*
* \param rm Event-queue resource manager
* \param instance Event-queue instance (index)
* \param dma_addrs Array to populate with addrs of allocated pages
* \param page_order Requested size of eventq
* \return < 0 if iobuffer allocation fails
*/
static int
efhw_nic_event_queue_alloc_iobuffer(struct efhw_nic *nic,
struct eventq_resource_hardware *h,
int evq_instance,
dma_addr_t *dma_addrs,
unsigned int page_order)
{
int i, j, rc;
/* Allocate an iobuffer. */
EFHW_TRACE("allocating eventq size %x",
1u << (page_order + PAGE_SHIFT));
rc = efhw_iopages_alloc(nic->pci_dev, &h->iobuff, page_order, NULL,
0UL);
if (rc < 0) {
EFHW_WARN("%s: failed to allocate %u pages",
__FUNCTION__, 1u << page_order);
return rc;
}
/* Set the eventq pages to match EFHW_CLEAR_EVENT() */
if (EFHW_CLEAR_EVENT_VALUE)
memset(efhw_iopages_ptr(&h->iobuff),
EFHW_CLEAR_EVENT_VALUE, (1u << page_order) * PAGE_SIZE);
EFHW_TRACE("%s: allocated %u pages", __FUNCTION__, 1u << (page_order));
/* For Falcon the NIC is programmed with the base buffer address of a
* contiguous region of buffer space. This means that larger than a
* PAGE event queues can be expected to allocate even when the host's
* physical memory is fragmented */
EFHW_ASSERT(efhw_nic_have_hw(nic));
EFHW_ASSERT(1 << EFHW_GFP_ORDER_TO_NIC_ORDER(page_order) <=
EFHW_BUFFER_TABLE_BLOCK_SIZE);
/* Initialise the buffer table entries. */
rc = efhw_nic_buffer_table_alloc(nic, 0, 0, &h->bt_block);
if (rc < 0) {
EFHW_WARN("%s: failed to allocate buffer table block",
__FUNCTION__);
efhw_iopages_free(nic->pci_dev, &h->iobuff, NULL);
return rc;
}
for (i = 0; i < (1 << page_order); ++i) {
for (j = 0; j < EFHW_NIC_PAGES_IN_OS_PAGE; ++j) {
dma_addrs[i * EFHW_NIC_PAGES_IN_OS_PAGE + j] =
efhw_iopages_dma_addr(&h->iobuff, i);
}
}
efhw_nic_buffer_table_set(nic, h->bt_block, 0,
1 << EFHW_GFP_ORDER_TO_NIC_ORDER(page_order),
dma_addrs);
falcon_nic_buffer_table_confirm(nic);
return 0;
}
void
efhw_handle_timeout_event(struct efhw_nic *nic, struct efhw_ev_handler *h,
unsigned instance)
{
if (!h->timeout_fn) {
EFHW_WARN("%s: no handler registered", __FUNCTION__);
return;
}
h->timeout_fn(nic, instance);
}
int
efhw_handle_wakeup_event(struct efhw_nic *nic, struct efhw_ev_handler *h,
unsigned instance, int budget)
{
if (!h->wakeup_fn) {
EFHW_WARN("%s: no handler registered", __FUNCTION__);
return 0;
}
return h->wakeup_fn(nic, instance, budget);
}
void
efhw_handle_timeout_event(struct efhw_nic *nic, struct efhw_ev_handler *h,
efhw_event_t *evp)
{
unsigned instance = (unsigned)FALCON_EVENT_WAKE_EVQ_ID(evp);
if (!h->timeout_fn) {
EFHW_WARN("%s: no handler registered", __FUNCTION__);
return;
}
h->timeout_fn(nic, instance);
}
int
efhw_handle_rxdmaq_flushed(struct efhw_nic *nic, struct efhw_ev_handler *h,
unsigned instance, int failed)
{
EFHW_TRACE("%s: instance=%d", __FUNCTION__, instance);
if (!h->dmaq_flushed_fn) {
EFHW_WARN("%s: no handler registered", __FUNCTION__);
return 0;
}
return h->dmaq_flushed_fn(nic, instance, true, failed);
}
void
efhw_handle_txdmaq_flushed(struct efhw_nic *nic, struct efhw_ev_handler *h,
efhw_event_t *evp)
{
int instance = (int)FALCON_EVENT_TX_FLUSH_Q_ID(evp);
EFHW_TRACE("%s: instance=%d", __FUNCTION__, instance);
if (!h->dmaq_flushed_fn) {
EFHW_WARN("%s: no handler registered", __FUNCTION__);
return;
}
h->dmaq_flushed_fn(nic, instance, false, false);
}
void
efhw_handle_rxdmaq_flushed(struct efhw_nic *nic, struct efhw_ev_handler *h,
efhw_event_t *evp)
{
unsigned instance = (unsigned)FALCON_EVENT_RX_FLUSH_Q_ID(evp);
unsigned failed = (int)FALCON_EVENT_RX_FLUSH_FAIL(evp);
EFHW_TRACE("%s: instance=%d", __FUNCTION__, instance);
if (!h->dmaq_flushed_fn) {
EFHW_WARN("%s: no handler registered", __FUNCTION__);
return;
}
h->dmaq_flushed_fn(nic, instance, true, failed);
}
/*! Helper function to allocate the iobuffer needed by an eventq
* - it ensures the eventq has the correct alignment for the NIC
*
* \param rm Event-queue resource manager
* \param instance Event-queue instance (index)
* \param buf_bytes Requested size of eventq
* \return < 0 if iobuffer allocation fails
*/
int
efhw_nic_event_queue_alloc_iobuffer(struct efhw_nic *nic,
struct eventq_resource_hardware *h,
int evq_instance, unsigned buf_bytes)
{
unsigned int page_order;
int rc;
/* Allocate an iobuffer. */
page_order = get_order(buf_bytes);
EFHW_TRACE("allocating eventq size %x",
1u << (page_order + PAGE_SHIFT));
rc = efhw_iopages_alloc(nic, &h->iobuff, page_order);
if (rc < 0) {
EFHW_WARN("%s: failed to allocate %u pages",
__FUNCTION__, 1u << page_order);
return rc;
}
/* Set the eventq pages to match EFHW_CLEAR_EVENT() */
if (EFHW_CLEAR_EVENT_VALUE)
memset(efhw_iopages_ptr(&h->iobuff),
EFHW_CLEAR_EVENT_VALUE, (1u << page_order) * PAGE_SIZE);
EFHW_TRACE("%s: allocated %u pages", __FUNCTION__, 1u << (page_order));
/* For Falcon the NIC is programmed with the base buffer address of a
* contiguous region of buffer space. This means that larger than a
* PAGE event queues can be expected to allocate even when the host's
* physical memory is fragmented */
EFHW_ASSERT(efhw_nic_have_hw(nic));
EFHW_ASSERT(page_order <= h->buf_tbl_alloc.order);
/* Initialise the buffer table entries. */
falcon_nic_buffer_table_set_n(nic, h->buf_tbl_alloc.base,
efhw_iopages_dma_addr(&h->iobuff),
EFHW_NIC_PAGE_SIZE, 0,
1 << page_order, 0);
falcon_nic_buffer_table_confirm(nic);
return 0;
}
int efhw_keventq_poll(struct efhw_nic *nic, struct efhw_keventq *q)
{
efhw_event_t *ev;
int l, count = 0;
EFHW_ASSERT(nic);
EFHW_ASSERT(q);
EFHW_ASSERT(q->ev_handlers);
/* Acquire the lock, or mark the queue as needing re-checking. */
for (;;) {
l = q->lock;
if (l == KEVQ_UNLOCKED) {
if ((int)cmpxchg(&q->lock, l, KEVQ_LOCKED) == l)
break;
} else if (l == KEVQ_LOCKED) {
if ((int)cmpxchg(&q->lock, l, KEVQ_RECHECK) == l)
return 0;
} else { /* already marked for re-checking */
EFHW_ASSERT(l == KEVQ_RECHECK);
return 0;
}
}
if (unlikely(EFHW_EVENT_OVERFLOW(q, q)))
goto overflow;
ev = EFHW_EVENT_PTR(q, q, 0);
#ifndef NDEBUG
if (!EFHW_IS_EVENT(ev))
EFHW_TRACE("%s: %d NO EVENTS!", __FUNCTION__, q->instance);
#endif
for (;;) {
/* Convention for return codes for handlers is:
** 0 - no error, event consumed
** 1 - no error, event not consumed
** -ve - error, event not consumed
*/
if (likely(EFHW_IS_EVENT(ev))) {
count++;
switch (FALCON_EVENT_CODE(ev)) {
case FALCON_EVENT_CODE_CHAR:
falcon_handle_char_event(nic, q->ev_handlers,
ev);
break;
default:
EFHW_ERR("efhw_keventq_poll: [%d] UNEXPECTED "
"EVENT:"FALCON_EVENT_FMT,
q->instance,
FALCON_EVENT_PRI_ARG(*ev));
}
EFHW_CLEAR_EVENT(ev);
EFHW_EVENTQ_NEXT(q);
ev = EFHW_EVENT_PTR(q, q, 0);
} else {
/* No events left. Release the lock (checking if we
* need to re-poll to avoid race). */
l = q->lock;
if (l == KEVQ_LOCKED) {
if ((int)cmpxchg(&q->lock, l, KEVQ_UNLOCKED)
== l) {
EFHW_TRACE
("efhw_keventq_poll: %d clean exit",
q->instance);
goto clean_exit;
}
}
/* Potentially more work to do. */
l = q->lock;
EFHW_ASSERT(l == KEVQ_RECHECK);
EFHW_TEST((int)cmpxchg(&q->lock, l, KEVQ_LOCKED) == l);
EFHW_TRACE("efhw_keventq_poll: %d re-poll required",
q->instance);
}
}
/* shouldn't get here */
EFHW_ASSERT(0);
overflow:
/* ?? Oh dear. Should we poll everything that could have possibly
** happened? Or merely cry out in anguish...
*/
EFHW_WARN("efhw_keventq_poll: %d ***** OVERFLOW nic %d *****",
q->instance, nic->index);
q->lock = KEVQ_UNLOCKED;
return count;
clean_exit:
/* Ack the processed events so that this event queue can potentially
raise interrupts again */
if (q->instance == nic->interrupting_evq.instance)
falcon_nic_evq_ack(nic, q->instance,
(EFHW_EVENT_OFFSET(q, q, 0)
/ sizeof(efhw_event_t)));
return count;
}
