static void copy_wr_to_sq(struct t4_wq *wq, union t4_wr *wqe, u8 len16)
{
u64 *src, *dst;
src = (u64 *)wqe;
dst = (u64 *)((u8 *)wq->sq.queue + wq->sq.wq_pidx * T4_EQ_ENTRY_SIZE);
if (t4_sq_onchip(wq)) {
len16 = align(len16, 4);
/* In onchip mode the copy below will be made to WC memory and
* could trigger DMA. In offchip mode the copy below only
* queues the WQE, DMA cannot start until t4_ring_sq_db
* happens */
mmio_wc_start();
}
while (len16) {
*dst++ = *src++;
if (dst == (u64 *)&wq->sq.queue[wq->sq.size])
dst = (u64 *)wq->sq.queue;
*dst++ = *src++;
if (dst == (u64 *)&wq->sq.queue[wq->sq.size])
dst = (u64 *)wq->sq.queue;
len16--;
/* NOTE len16 cannot be large enough to write to the
same sq.queue memory twice in this loop */
}
if (t4_sq_onchip(wq))
mmio_flush_writes();
}
static void copy_wr_to_sq(struct t4_wq *wq, union t4_wr *wqe, u8 len16)
{
void *src, *dst;
uintptr_t end;
int total, len;
src = &wqe->flits[0];
dst = &wq->sq.queue->flits[wq->sq.wq_pidx *
(T4_EQ_ENTRY_SIZE / sizeof(__be64))];
if (t4_sq_onchip(wq)) {
len16 = align(len16, 4);
/* In onchip mode the copy below will be made to WC memory and
* could trigger DMA. In offchip mode the copy below only
* queues the WQE, DMA cannot start until t4_ring_sq_db
* happens */
mmio_wc_start();
}
/* NOTE len16 cannot be large enough to write to the
same sq.queue memory twice in this loop */
total = len16 * 16;
end = (uintptr_t)&wq->sq.queue[wq->sq.size];
if (__predict_true((uintptr_t)dst + total <= end)) {
/* Won't wrap around. */
memcpy(dst, src, total);
} else {
len = end - (uintptr_t)dst;
memcpy(dst, src, len);
memcpy(wq->sq.queue, src + len, total - len);
}
if (t4_sq_onchip(wq))
mmio_flush_writes();
}