static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
struct efx_tx_buffer *buffer)
{
if (buffer->unmap_len) {
struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
dma_addr_t unmap_addr = (buffer->dma_addr + buffer->len -
buffer->unmap_len);
if (buffer->unmap_single)
pci_unmap_single(pci_dev, unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(pci_dev, unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
buffer->unmap_len = 0;
buffer->unmap_single = false;
}
if (buffer->skb) {
dev_kfree_skb_any((struct sk_buff *) buffer->skb);
buffer->skb = NULL;
netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,
"TX queue %d transmission id %x complete\n",
tx_queue->queue, tx_queue->read_count);
}
}
static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
ttm_agp_tt_unpopulate(ttm);
return;
}
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
ttm_dma_unpopulate(>t->ttm, rdev->dev);
return;
}
#endif
for (i = 0; i < ttm->num_pages; i++) {
if (gtt->ttm.dma_address[i]) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
}
}
ttm_pool_unpopulate(ttm);
}
void iwl_rx_queue_reset(struct iwl_priv *priv, struct iwl_rx_queue *rxq)
{
unsigned long flags;
int i;
spin_lock_irqsave(&rxq->lock, flags);
INIT_LIST_HEAD(&rxq->rx_free);
INIT_LIST_HEAD(&rxq->rx_used);
/* Fill the rx_used queue with _all_ of the Rx buffers */
for (i = 0; i < RX_FREE_BUFFERS + RX_QUEUE_SIZE; i++) {
/* In the reset function, these buffers may have been allocated
* to an SKB, so we need to unmap and free potential storage */
if (rxq->pool[i].page != NULL) {
pci_unmap_page(priv->pci_dev, rxq->pool[i].page_dma,
PAGE_SIZE << priv->hw_params.rx_page_order,
PCI_DMA_FROMDEVICE);
__iwl_free_pages(priv, rxq->pool[i].page);
rxq->pool[i].page = NULL;
}
list_add_tail(&rxq->pool[i].list, &rxq->rx_used);
}
/* Set us so that we have processed and used all buffers, but have
* not restocked the Rx queue with fresh buffers */
rxq->read = rxq->write = 0;
rxq->write_actual = 0;
rxq->free_count = 0;
spin_unlock_irqrestore(&rxq->lock, flags);
}
void netxen_release_tx_buffers(struct netxen_adapter *adapter)
{
struct netxen_cmd_buffer *cmd_buf;
struct netxen_skb_frag *buffrag;
int i, j;
struct nx_host_tx_ring *tx_ring = adapter->tx_ring;
cmd_buf = tx_ring->cmd_buf_arr;
for (i = 0; i < tx_ring->num_desc; i++) {
buffrag = cmd_buf->frag_array;
if (buffrag->dma) {
pci_unmap_single(adapter->pdev, buffrag->dma,
buffrag->length, PCI_DMA_TODEVICE);
buffrag->dma = 0ULL;
}
for (j = 0; j < cmd_buf->frag_count; j++) {
buffrag++;
if (buffrag->dma) {
pci_unmap_page(adapter->pdev, buffrag->dma,
buffrag->length,
PCI_DMA_TODEVICE);
buffrag->dma = 0ULL;
}
}
if (cmd_buf->skb) {
dev_kfree_skb_any(cmd_buf->skb);
cmd_buf->skb = NULL;
}
cmd_buf++;
}
}
void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
{
int i;
struct efx_rx_buffer *rx_buf;
EFX_LOG(rx_queue->efx, "shutting down RX queue %d\n", rx_queue->queue);
falcon_fini_rx(rx_queue);
/* Release RX buffers NB start at index 0 not current HW ptr */
if (rx_queue->buffer) {
for (i = 0; i <= rx_queue->efx->type->rxd_ring_mask; i++) {
rx_buf = efx_rx_buffer(rx_queue, i);
efx_fini_rx_buffer(rx_queue, rx_buf);
}
}
/* For a page that is part-way through splitting into RX buffers */
if (rx_queue->buf_page != NULL) {
pci_unmap_page(rx_queue->efx->pci_dev, rx_queue->buf_dma_addr,
efx_rx_buf_size(rx_queue->efx),
PCI_DMA_FROMDEVICE);
__free_pages(rx_queue->buf_page,
rx_queue->efx->rx_buffer_order);
rx_queue->buf_page = NULL;
}
}
/* Remove descriptors put into a tx_queue. */
static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
{
struct efx_tx_buffer *buffer;
dma_addr_t unmap_addr;
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
buffer = &tx_queue->buffer[tx_queue->insert_count &
tx_queue->ptr_mask];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->skb);
if (buffer->unmap_len) {
unmap_addr = (buffer->dma_addr + buffer->len -
buffer->unmap_len);
if (buffer->unmap_single)
pci_unmap_single(tx_queue->efx->pci_dev,
unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(tx_queue->efx->pci_dev,
unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
buffer->unmap_len = 0;
}
buffer->len = 0;
buffer->continuation = true;
}
}
/**
* amdgpu_dummy_page_fini - free dummy page used by the driver
*
* @adev: amdgpu_device pointer
*
* Frees the dummy page used by the driver (all asics).
*/
static void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev)
{
if (!adev->dummy_page_addr)
return;
pci_unmap_page(adev->pdev, adev->dummy_page_addr,
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
adev->dummy_page_addr = 0;
}
static void
unmap_ddp_gl(const struct ddp_gather_list *gl)
{
#ifdef NEED_BUSDMA
int i;
if (!gl->nelem)
return;
pci_unmap_page(pdev, gl->phys_addr[0] + gl->offset,
PAGE_SIZE - gl->offset, PCI_DMA_FROMDEVICE);
for (i = 1; i < gl->nelem; ++i)
pci_unmap_page(pdev, gl->phys_addr[i], PAGE_SIZE,
PCI_DMA_FROMDEVICE);
#endif
}
void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
enum dma_data_direction direction)
{
if (dev->bus == &pci_bus_type)
pci_unmap_page(to_pci_dev(dev), dma_address, size, (int)direction);
else if (dev->bus == &vio_bus_type)
vio_unmap_page(to_vio_dev(dev), dma_address, size, direction);
else
BUG();
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r;
bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
if (ttm->state != tt_unpopulated)
return 0;
if (slave && ttm->sg) {
drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
gtt->ttm.dma_address, ttm->num_pages);
ttm->state = tt_unbound;
return 0;
}
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i],
0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) {
while (--i) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
gtt->ttm.dma_address[i] = 0;
}
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
}
return 0;
}
static void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf)
{
struct enic *enic = vnic_dev_priv(wq->vdev);
if (buf->sop)
pci_unmap_single(enic->pdev, buf->dma_addr,
buf->len, PCI_DMA_TODEVICE);
else
pci_unmap_page(enic->pdev, buf->dma_addr,
buf->len, PCI_DMA_TODEVICE);
if (buf->os_buf)
dev_kfree_skb_any(buf->os_buf);
}
dma_addr_t ipath_map_page(struct pci_dev *hwdev, struct page *page,
unsigned long offset, size_t size, int direction)
{
dma_addr_t phys;
phys = pci_map_page(hwdev, page, offset, size, direction);
if (phys == 0) {
pci_unmap_page(hwdev, phys, size, direction);
phys = pci_map_page(hwdev, page, offset, size, direction);
}
return phys;
}
static void efx_unmap_rx_buffer(struct efx_nic *efx,
struct efx_rx_buffer *rx_buf)
{
if (rx_buf->page) {
EFX_BUG_ON_PARANOID(rx_buf->skb);
if (rx_buf->unmap_addr) {
pci_unmap_page(efx->pci_dev, rx_buf->unmap_addr,
efx_rx_buf_size(efx),
PCI_DMA_FROMDEVICE);
rx_buf->unmap_addr = 0;
}
} else if (likely(rx_buf->skb)) {
pci_unmap_single(efx->pci_dev, rx_buf->dma_addr,
rx_buf->len, PCI_DMA_FROMDEVICE);
}
}
static void
nvc0_fb_destroy(struct drm_device *dev)
{
struct drm_nouveau_private *dev_priv = dev->dev_private;
struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
struct nvc0_fb_priv *priv = pfb->priv;
nouveau_irq_unregister(dev, 25);
if (priv->r100c10_page) {
pci_unmap_page(dev->pdev, priv->r100c10, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
__free_page(priv->r100c10_page);
}
kfree(priv);
pfb->priv = NULL;
}
/**
* qib_map_page - a safety wrapper around pci_map_page()
*
* A dma_addr of all 0's is interpreted by the chip as "disabled".
* Unfortunately, it can also be a valid dma_addr returned on some
* architectures.
*
* The powerpc iommu assigns dma_addrs in ascending order, so we don't
* have to bother with retries or mapping a dummy page to insure we
* don't just get the same mapping again.
*
* I'm sure we won't be so lucky with other iommu's, so FIXME.
*/
dma_addr_t qib_map_page(struct pci_dev *hwdev, struct page *page,
unsigned long offset, size_t size, int direction)
{
dma_addr_t phys;
phys = pci_map_page(hwdev, page, offset, size, direction);
if (phys == 0) {
pci_unmap_page(hwdev, phys, size, direction);
phys = pci_map_page(hwdev, page, offset, size, direction);
/*
* FIXME: If we get 0 again, we should keep this page,
* map another, then free the 0 page.
*/
}
return phys;
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r;
if (ttm->state != tt_unpopulated)
return 0;
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i],
0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) {
while (--i) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
gtt->ttm.dma_address[i] = 0;
}
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
}
return 0;
}
static void be_rx_q_clean(struct be_net_object *pnob)
{
if (pnob->rx_ctxt) {
int i;
struct be_rx_page_info *rx_page_info;
for (i = 0; i < pnob->rx_q_len; i++) {
rx_page_info = &(pnob->rx_page_info[i]);
if (!pnob->rx_pg_shared || rx_page_info->page_offset) {
pci_unmap_page(pnob->adapter->pdev,
pci_unmap_addr(rx_page_info, bus),
pnob->rx_buf_size,
PCI_DMA_FROMDEVICE);
}
if (rx_page_info->page)
put_page(rx_page_info->page);
memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
}
pnob->rx_pg_info_hd = 0;
}
}
void dl_dma_unmap_kernel_buffer(struct dl_dma_list* sl, int direction)
{
unsigned long i;
struct dl_dma_entry *e = first_entry(sl);
direction = bmd_to_linux_direction(direction);
if (!sl->dma_is_single)
{
for (i = 0; i < sl->num_pages; i++)
{
pci_unmap_page(sl->pdev, e->dma_addr, PAGE_SIZE, direction);
e = next_entry(e);
}
}
else
pci_unmap_single(sl->pdev, e->dma_addr, sl->size, direction);
destroy_dl_dma_entry(sl);
}
/* Assumes that the skb field of the buffers in 'pool' is kept accurate.
* If an SKB has been detached, the POOL needs to have its SKB set to NULL
* This free routine walks the list of POOL entries and if SKB is set to
* non NULL it is unmapped and freed
*/
void iwl_rx_queue_free(struct iwl_priv *priv, struct iwl_rx_queue *rxq)
{
int i;
for (i = 0; i < RX_QUEUE_SIZE + RX_FREE_BUFFERS; i++) {
if (rxq->pool[i].page != NULL) {
pci_unmap_page(priv->pci_dev, rxq->pool[i].page_dma,
PAGE_SIZE << priv->hw_params.rx_page_order,
PCI_DMA_FROMDEVICE);
__iwl_free_pages(priv, rxq->pool[i].page);
rxq->pool[i].page = NULL;
}
}
dma_free_coherent(&priv->pci_dev->dev, 4 * RX_QUEUE_SIZE, rxq->bd,
rxq->dma_addr);
dma_free_coherent(&priv->pci_dev->dev, sizeof(struct iwl_rb_status),
rxq->rb_stts, rxq->rb_stts_dma);
rxq->bd = NULL;
rxq->rb_stts = NULL;
}
void
efx_vi_dma_unmap_pages(struct efx_vi_state *vih,
struct efx_vi_dma_map_state *dmh)
{
struct efx_vi_state *efx_state = vih;
struct efx_vi_dma_map_state *dm_state =
(struct efx_vi_dma_map_state *)dmh;
int i;
efrm_buffer_table_free(&dm_state->bt_handle);
for (i = 0; i < dm_state->n_pages; ++i)
pci_unmap_page(linux_efhw_nic(efx_state->nic)->pci_dev,
dm_state->dma_addrs[i], PAGE_SIZE,
PCI_DMA_TODEVICE);
kfree(dm_state->dma_addrs);
kfree(dm_state);
return;
}
static inline void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
struct efx_tx_buffer *buffer)
{
if (buffer->unmap_len) {
struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
if (buffer->unmap_single)
pci_unmap_single(pci_dev, buffer->unmap_addr,
buffer->unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(pci_dev, buffer->unmap_addr,
buffer->unmap_len, PCI_DMA_TODEVICE);
buffer->unmap_len = 0;
buffer->unmap_single = 0;
}
if (buffer->skb) {
dev_kfree_skb_any((struct sk_buff *) buffer->skb);
buffer->skb = NULL;
EFX_TRACE(tx_queue->efx, "TX queue %d transmission id %x "
"complete\n", tx_queue->queue, read_ptr);
}
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r;
#ifdef DUMBBELL_WIP
bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
#endif /* DUMBBELL_WIP */
if (ttm->state != tt_unpopulated)
return 0;
#ifdef DUMBBELL_WIP
/*
* Maybe unneeded on FreeBSD.
* -- dumbbell@
*/
if (slave && ttm->sg) {
drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
gtt->ttm.dma_address, ttm->num_pages);
ttm->state = tt_unbound;
return 0;
}
#endif /* DUMBBELL_WIP */
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
#ifdef DUMBBELL_WIP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif /* DUMBBELL_WIP */
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->ttm.dma_address[i] = VM_PAGE_TO_PHYS(ttm->pages[i]);
#ifdef DUMBBELL_WIP
gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i],
0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) {
while (--i) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
gtt->ttm.dma_address[i] = 0;
}
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
#endif /* DUMBBELL_WIP */
}
return 0;
}
/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
/* Since the stack does not limit the number of segments per
* skb, we must do so. Otherwise an attacker may be able to
* make the TCP produce skbs that will never fit in our TX
* queue, causing repeated resets.
*/
if (unlikely(skb_shinfo(skb)->gso_segs > EFX_TSO_MAX_SEGS)) {
unsigned int excess =
(skb_shinfo(skb)->gso_segs - EFX_TSO_MAX_SEGS) *
skb_shinfo(skb)->gso_size;
if (__pskb_trim(skb, skb->len - excess)) {
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
}
/* Find the packet protocol and sanity-check it */
state.protocol = efx_tso_check_protocol(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.header_len) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
rc = tso_get_head_fragment(&state, efx, skb);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc)) {
rc2 = NETDEV_TX_BUSY;
goto unwind;
}
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or PCI mapping error\n");
dev_kfree_skb_any(skb);
unwind:
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.unmap_single)
pci_unmap_single(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
}
efx_enqueue_unwind(tx_queue);
return rc2;
}
/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
/* Find the packet protocol and sanity-check it */
state.protocol = efx_tso_check_protocol(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.header_len) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
rc = tso_get_head_fragment(&state, efx, skb);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc)) {
rc2 = NETDEV_TX_BUSY;
goto unwind;
}
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or PCI mapping error\n");
dev_kfree_skb_any(skb);
unwind:
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.unmap_single)
pci_unmap_single(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
}
efx_enqueue_unwind(tx_queue);
return rc2;
}
void inline pci64_unmap_page(struct pci_dev *hwdev, dma_addr_t address,
size_t size, int direction)
{
pci_unmap_page(hwdev, address, size, direction);
}
/*
* Add a socket buffer to a TX queue
*
* This maps all fragments of a socket buffer for DMA and adds them to
* the TX queue. The queue's insert pointer will be incremented by
* the number of fragments in the socket buffer.
*
* If any DMA mapping fails, any mapped fragments will be unmapped,
* the queue's insert pointer will be restored to its original value.
*
* This function is split out from efx_hard_start_xmit to allow the
* loopback test to direct packets via specific TX queues.
*
* Returns NETDEV_TX_OK or NETDEV_TX_BUSY
* You must hold netif_tx_lock() to call this function.
*/
netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
struct pci_dev *pci_dev = efx->pci_dev;
struct efx_tx_buffer *buffer;
skb_frag_t *fragment;
struct page *page;
int page_offset;
unsigned int len, unmap_len = 0, fill_level, insert_ptr;
dma_addr_t dma_addr, unmap_addr = 0;
unsigned int dma_len;
bool unmap_single;
int q_space, i = 0;
netdev_tx_t rc = NETDEV_TX_OK;
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
if (skb_shinfo(skb)->gso_size)
return efx_enqueue_skb_tso(tx_queue, skb);
/* Get size of the initial fragment */
len = skb_headlen(skb);
/* Pad if necessary */
if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
EFX_BUG_ON_PARANOID(skb->data_len);
len = 32 + 1;
if (skb_pad(skb, len - skb->len))
return NETDEV_TX_OK;
}
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
q_space = efx->txq_entries - 1 - fill_level;
/* Map for DMA. Use pci_map_single rather than pci_map_page
* since this is more efficient on machines with sparse
* memory.
*/
unmap_single = true;
dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
/* Process all fragments */
while (1) {
if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
goto pci_err;
/* Store fields for marking in the per-fragment final
* descriptor */
unmap_len = len;
unmap_addr = dma_addr;
/* Add to TX queue, splitting across DMA boundaries */
do {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have
* happened since the xmit path last
* checked. Update the xmit path's
* copy of read_count.
*/
netif_tx_stop_queue(tx_queue->core_txq);
/* This memory barrier protects the
* change of queue state from the access
* of read_count. */
smp_mb();
tx_queue->old_read_count =
ACCESS_ONCE(tx_queue->read_count);
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = efx->txq_entries - 1 - fill_level;
if (unlikely(q_space-- <= 0)) {
rc = NETDEV_TX_BUSY;
goto unwind;
}
smp_mb();
netif_tx_start_queue(tx_queue->core_txq);
}
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->tsoh);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
dma_len = efx_max_tx_len(efx, dma_addr);
if (likely(dma_len >= len))
dma_len = len;
/* Fill out per descriptor fields */
buffer->len = dma_len;
buffer->dma_addr = dma_addr;
len -= dma_len;
dma_addr += dma_len;
++tx_queue->insert_count;
} while (len);
/* Transfer ownership of the unmapping to the final buffer */
buffer->unmap_single = unmap_single;
buffer->unmap_len = unmap_len;
unmap_len = 0;
/* Get address and size of next fragment */
if (i >= skb_shinfo(skb)->nr_frags)
break;
fragment = &skb_shinfo(skb)->frags[i];
len = fragment->size;
page = fragment->page;
page_offset = fragment->page_offset;
i++;
/* Map for DMA */
unmap_single = false;
dma_addr = pci_map_page(pci_dev, page, page_offset, len,
PCI_DMA_TODEVICE);
}
/* Transfer ownership of the skb to the final buffer */
buffer->skb = skb;
buffer->continuation = false;
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
return NETDEV_TX_OK;
pci_err:
netif_err(efx, tx_err, efx->net_dev,
" TX queue %d could not map skb with %d bytes %d "
"fragments for DMA\n", tx_queue->queue, skb->len,
skb_shinfo(skb)->nr_frags + 1);
/* Mark the packet as transmitted, and free the SKB ourselves */
dev_kfree_skb_any(skb);
unwind:
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_dequeue_buffer(tx_queue, buffer);
buffer->len = 0;
}
/* Free the fragment we were mid-way through pushing */
if (unmap_len) {
if (unmap_single)
pci_unmap_single(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
}
return rc;
}
/*
* Add a socket buffer to a TX queue
*
* This maps all fragments of a socket buffer for DMA and adds them to
* the TX queue. The queue's insert pointer will be incremented by
* the number of fragments in the socket buffer.
*
* If any DMA mapping fails, any mapped fragments will be unmapped,
* the queue's insert pointer will be restored to its original value.
*
* Returns NETDEV_TX_OK or NETDEV_TX_BUSY
* You must hold netif_tx_lock() to call this function.
*/
static inline int efx_enqueue_skb(struct efx_tx_queue *tx_queue,
const struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
struct pci_dev *pci_dev = efx->pci_dev;
struct efx_tx_buffer *buffer;
skb_frag_t *fragment;
struct page *page;
int page_offset;
unsigned int len, unmap_len = 0, fill_level, insert_ptr, misalign;
dma_addr_t dma_addr, unmap_addr = 0;
unsigned int dma_len;
unsigned unmap_single;
int q_space, i = 0;
int rc = NETDEV_TX_OK;
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
if (skb_shinfo((struct sk_buff *)skb)->gso_size)
return efx_enqueue_skb_tso(tx_queue, skb);
/* Get size of the initial fragment */
len = skb_headlen(skb);
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
q_space = efx->type->txd_ring_mask - 1 - fill_level;
/* Map for DMA. Use pci_map_single rather than pci_map_page
* since this is more efficient on machines with sparse
* memory.
*/
unmap_single = 1;
dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
/* Process all fragments */
while (1) {
if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
goto pci_err;
/* Store fields for marking in the per-fragment final
* descriptor */
unmap_len = len;
unmap_addr = dma_addr;
/* Add to TX queue, splitting across DMA boundaries */
do {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have
* happened since the xmit path last
* checked. Update the xmit path's
* copy of read_count.
*/
++tx_queue->stopped;
/* This memory barrier protects the
* change of stopped from the access
* of read_count. */
smp_mb();
tx_queue->old_read_count =
*(volatile unsigned *)
&tx_queue->read_count;
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = (efx->type->txd_ring_mask - 1 -
fill_level);
if (unlikely(q_space-- <= 0))
goto stop;
smp_mb();
--tx_queue->stopped;
}
insert_ptr = (tx_queue->insert_count &
efx->type->txd_ring_mask);
buffer = &tx_queue->buffer[insert_ptr];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->tsoh);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->continuation != 1);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
dma_len = (((~dma_addr) & efx->type->tx_dma_mask) + 1);
if (likely(dma_len > len))
dma_len = len;
misalign = (unsigned)dma_addr & efx->type->bug5391_mask;
if (misalign && dma_len + misalign > 512)
dma_len = 512 - misalign;
/* Fill out per descriptor fields */
buffer->len = dma_len;
buffer->dma_addr = dma_addr;
len -= dma_len;
dma_addr += dma_len;
++tx_queue->insert_count;
} while (len);
/* Transfer ownership of the unmapping to the final buffer */
buffer->unmap_addr = unmap_addr;
buffer->unmap_single = unmap_single;
buffer->unmap_len = unmap_len;
unmap_len = 0;
/* Get address and size of next fragment */
if (i >= skb_shinfo(skb)->nr_frags)
break;
fragment = &skb_shinfo(skb)->frags[i];
len = fragment->size;
page = fragment->page;
page_offset = fragment->page_offset;
i++;
/* Map for DMA */
unmap_single = 0;
dma_addr = pci_map_page(pci_dev, page, page_offset, len,
PCI_DMA_TODEVICE);
}
/* Transfer ownership of the skb to the final buffer */
buffer->skb = skb;
buffer->continuation = 0;
/* Pass off to hardware */
falcon_push_buffers(tx_queue);
return NETDEV_TX_OK;
pci_err:
EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d "
"fragments for DMA\n", tx_queue->queue, skb->len,
skb_shinfo(skb)->nr_frags + 1);
/* Mark the packet as transmitted, and free the SKB ourselves */
dev_kfree_skb_any((struct sk_buff *)skb);
goto unwind;
stop:
rc = NETDEV_TX_BUSY;
if (tx_queue->stopped == 1)
efx_stop_queue(efx);
unwind:
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_dequeue_buffer(tx_queue, buffer);
buffer->len = 0;
}
/* Free the fragment we were mid-way through pushing */
if (unmap_len)
pci_unmap_page(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
return rc;
}
static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
int index, u8 owner)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset;
struct sk_buff *skb = tx_info->skb;
struct skb_frag_struct *frag;
void *end = ring->buf + ring->buf_size;
int frags = skb_shinfo(skb)->nr_frags;
int i;
__be32 *ptr = (__be32 *)tx_desc;
__be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT));
/* Optimize the common case when there are no wraparounds */
if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
if (!tx_info->inl) {
if (tx_info->linear) {
pci_unmap_single(mdev->pdev,
(dma_addr_t) be64_to_cpu(data->addr),
be32_to_cpu(data->byte_count),
PCI_DMA_TODEVICE);
++data;
}
for (i = 0; i < frags; i++) {
frag = &skb_shinfo(skb)->frags[i];
pci_unmap_page(mdev->pdev,
(dma_addr_t) be64_to_cpu(data[i].addr),
frag->size, PCI_DMA_TODEVICE);
}
}
/* Stamp the freed descriptor */
for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) {
*ptr = stamp;
ptr += STAMP_DWORDS;
}
} else {
if (!tx_info->inl) {
if ((void *) data >= end) {
data = (struct mlx4_wqe_data_seg *)
(ring->buf + ((void *) data - end));
}
if (tx_info->linear) {
pci_unmap_single(mdev->pdev,
(dma_addr_t) be64_to_cpu(data->addr),
be32_to_cpu(data->byte_count),
PCI_DMA_TODEVICE);
++data;
}
for (i = 0; i < frags; i++) {
/* Check for wraparound before unmapping */
if ((void *) data >= end)
data = (struct mlx4_wqe_data_seg *) ring->buf;
frag = &skb_shinfo(skb)->frags[i];
pci_unmap_page(mdev->pdev,
(dma_addr_t) be64_to_cpu(data->addr),
frag->size, PCI_DMA_TODEVICE);
++data;
}
}
/* Stamp the freed descriptor */
for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) {
*ptr = stamp;
ptr += STAMP_DWORDS;
if ((void *) ptr >= end) {
ptr = ring->buf;
stamp ^= cpu_to_be32(0x80000000);
}
}
}
dev_kfree_skb_any(skb);
return tx_info->nr_txbb;
}
/**
* qib_tid_free - free a context TID
* @rcd: the context
* @subctxt: the subcontext
* @ti: the TID info
*
* right now we are unlocking one page at a time, but since
* the intended use of this routine is for a single group of
* virtually contiguous pages, that should change to improve
* performance. We check that the TID is in range for this context
* but otherwise don't check validity; if user has an error and
* frees the wrong tid, it's only their own data that can thereby
* be corrupted. We do check that the TID was in use, for sanity
* We always use our idea of the saved address, not the address that
* they pass in to us.
*/
static int qib_tid_free(struct qib_ctxtdata *rcd, unsigned subctxt,
const struct qib_tid_info *ti)
{
int ret = 0;
u32 tid, ctxttid, cnt, limit, tidcnt;
struct qib_devdata *dd = rcd->dd;
u64 __iomem *tidbase;
unsigned long tidmap[8];
if (!dd->pageshadow) {
ret = -ENOMEM;
goto done;
}
if (copy_from_user(tidmap, (void __user *)(unsigned long)ti->tidmap,
sizeof tidmap)) {
ret = -EFAULT;
goto done;
}
ctxttid = rcd->ctxt * dd->rcvtidcnt;
if (!rcd->subctxt_cnt)
tidcnt = dd->rcvtidcnt;
else if (!subctxt) {
tidcnt = (dd->rcvtidcnt / rcd->subctxt_cnt) +
(dd->rcvtidcnt % rcd->subctxt_cnt);
ctxttid += dd->rcvtidcnt - tidcnt;
} else {
tidcnt = dd->rcvtidcnt / rcd->subctxt_cnt;
ctxttid += tidcnt * (subctxt - 1);
}
tidbase = (u64 __iomem *) ((char __iomem *)(dd->kregbase) +
dd->rcvtidbase +
ctxttid * sizeof(*tidbase));
limit = sizeof(tidmap) * BITS_PER_BYTE;
if (limit > tidcnt)
/* just in case size changes in future */
limit = tidcnt;
tid = find_first_bit(tidmap, limit);
for (cnt = 0; tid < limit; tid++) {
/*
* small optimization; if we detect a run of 3 or so without
* any set, use find_first_bit again. That's mainly to
* accelerate the case where we wrapped, so we have some at
* the beginning, and some at the end, and a big gap
* in the middle.
*/
if (!test_bit(tid, tidmap))
continue;
cnt++;
if (dd->pageshadow[ctxttid + tid]) {
struct page *p;
dma_addr_t phys;
p = dd->pageshadow[ctxttid + tid];
dd->pageshadow[ctxttid + tid] = NULL;
phys = dd->physshadow[ctxttid + tid];
dd->physshadow[ctxttid + tid] = dd->tidinvalid;
/* PERFORMANCE: below should almost certainly be
* cached
*/
dd->f_put_tid(dd, &tidbase[tid],
RCVHQ_RCV_TYPE_EXPECTED, dd->tidinvalid);
pci_unmap_page(dd->pcidev, phys, PAGE_SIZE,
PCI_DMA_FROMDEVICE);
qib_release_user_pages(&p, 1);
}
}
done:
return ret;
}
/**
* qib_tid_update - update a context TID
* @rcd: the context
* @fp: the qib device file
* @ti: the TID information
*
* The new implementation as of Oct 2004 is that the driver assigns
* the tid and returns it to the caller. To reduce search time, we
* keep a cursor for each context, walking the shadow tid array to find
* one that's not in use.
*
* For now, if we can't allocate the full list, we fail, although
* in the long run, we'll allocate as many as we can, and the
* caller will deal with that by trying the remaining pages later.
* That means that when we fail, we have to mark the tids as not in
* use again, in our shadow copy.
*
* It's up to the caller to free the tids when they are done.
* We'll unlock the pages as they free them.
*
* Also, right now we are locking one page at a time, but since
* the intended use of this routine is for a single group of
* virtually contiguous pages, that should change to improve
* performance.
*/
static int qib_tid_update(struct qib_ctxtdata *rcd, struct file *fp,
const struct qib_tid_info *ti)
{
int ret = 0, ntids;
u32 tid, ctxttid, cnt, i, tidcnt, tidoff;
u16 *tidlist;
struct qib_devdata *dd = rcd->dd;
u64 physaddr;
unsigned long vaddr;
u64 __iomem *tidbase;
unsigned long tidmap[8];
struct page **pagep = NULL;
unsigned subctxt = subctxt_fp(fp);
if (!dd->pageshadow) {
ret = -ENOMEM;
goto done;
}
cnt = ti->tidcnt;
if (!cnt) {
ret = -EFAULT;
goto done;
}
ctxttid = rcd->ctxt * dd->rcvtidcnt;
if (!rcd->subctxt_cnt) {
tidcnt = dd->rcvtidcnt;
tid = rcd->tidcursor;
tidoff = 0;
} else if (!subctxt) {
tidcnt = (dd->rcvtidcnt / rcd->subctxt_cnt) +
(dd->rcvtidcnt % rcd->subctxt_cnt);
tidoff = dd->rcvtidcnt - tidcnt;
ctxttid += tidoff;
tid = tidcursor_fp(fp);
} else {
tidcnt = dd->rcvtidcnt / rcd->subctxt_cnt;
tidoff = tidcnt * (subctxt - 1);
ctxttid += tidoff;
tid = tidcursor_fp(fp);
}
if (cnt > tidcnt) {
/* make sure it all fits in tid_pg_list */
qib_devinfo(dd->pcidev,
"Process tried to allocate %u TIDs, only trying max (%u)\n",
cnt, tidcnt);
cnt = tidcnt;
}
pagep = (struct page **) rcd->tid_pg_list;
tidlist = (u16 *) &pagep[dd->rcvtidcnt];
pagep += tidoff;
tidlist += tidoff;
memset(tidmap, 0, sizeof(tidmap));
/* before decrement; chip actual # */
ntids = tidcnt;
tidbase = (u64 __iomem *) (((char __iomem *) dd->kregbase) +
dd->rcvtidbase +
ctxttid * sizeof(*tidbase));
/* virtual address of first page in transfer */
vaddr = ti->tidvaddr;
if (!access_ok(VERIFY_WRITE, (void __user *) vaddr,
cnt * PAGE_SIZE)) {
ret = -EFAULT;
goto done;
}
ret = qib_get_user_pages(vaddr, cnt, pagep);
if (ret) {
/*
* if (ret == -EBUSY)
* We can't continue because the pagep array won't be
* initialized. This should never happen,
* unless perhaps the user has mpin'ed the pages
* themselves.
*/
qib_devinfo(dd->pcidev,
"Failed to lock addr %p, %u pages: "
"errno %d\n", (void *) vaddr, cnt, -ret);
goto done;
}
for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) {
for (; ntids--; tid++) {
if (tid == tidcnt)
tid = 0;
if (!dd->pageshadow[ctxttid + tid])
break;
}
if (ntids < 0) {
/*
* Oops, wrapped all the way through their TIDs,
* and didn't have enough free; see comments at
* start of routine
*/
i--; /* last tidlist[i] not filled in */
ret = -ENOMEM;
break;
}
tidlist[i] = tid + tidoff;
/* we "know" system pages and TID pages are same size */
dd->pageshadow[ctxttid + tid] = pagep[i];
dd->physshadow[ctxttid + tid] =
qib_map_page(dd->pcidev, pagep[i], 0, PAGE_SIZE,
PCI_DMA_FROMDEVICE);
/*
* don't need atomic or it's overhead
*/
__set_bit(tid, tidmap);
physaddr = dd->physshadow[ctxttid + tid];
/* PERFORMANCE: below should almost certainly be cached */
dd->f_put_tid(dd, &tidbase[tid],
RCVHQ_RCV_TYPE_EXPECTED, physaddr);
/*
* don't check this tid in qib_ctxtshadow, since we
* just filled it in; start with the next one.
*/
tid++;
}
if (ret) {
u32 limit;
cleanup:
/* jump here if copy out of updated info failed... */
/* same code that's in qib_free_tid() */
limit = sizeof(tidmap) * BITS_PER_BYTE;
if (limit > tidcnt)
/* just in case size changes in future */
limit = tidcnt;
tid = find_first_bit((const unsigned long *)tidmap, limit);
for (; tid < limit; tid++) {
if (!test_bit(tid, tidmap))
continue;
if (dd->pageshadow[ctxttid + tid]) {
dma_addr_t phys;
phys = dd->physshadow[ctxttid + tid];
dd->physshadow[ctxttid + tid] = dd->tidinvalid;
/* PERFORMANCE: below should almost certainly
* be cached
*/
dd->f_put_tid(dd, &tidbase[tid],
RCVHQ_RCV_TYPE_EXPECTED,
dd->tidinvalid);
pci_unmap_page(dd->pcidev, phys, PAGE_SIZE,
PCI_DMA_FROMDEVICE);
dd->pageshadow[ctxttid + tid] = NULL;
}
}
qib_release_user_pages(pagep, cnt);
} else {
/*
* Copy the updated array, with qib_tid's filled in, back
* to user. Since we did the copy in already, this "should
* never fail" If it does, we have to clean up...
*/
if (copy_to_user((void __user *)
(unsigned long) ti->tidlist,
tidlist, cnt * sizeof(*tidlist))) {
ret = -EFAULT;
goto cleanup;
}
if (copy_to_user((void __user *) (unsigned long) ti->tidmap,
tidmap, sizeof tidmap)) {
ret = -EFAULT;
goto cleanup;
}
if (tid == tidcnt)
tid = 0;
if (!rcd->subctxt_cnt)
rcd->tidcursor = tid;
else
tidcursor_fp(fp) = tid;
}
done:
return ret;
}