/** * ipath_map_page - a safety wrapper around pci_map_page() * * A dma_addr of all 0's is interpreted by the chip as "disabled". * Unfortunately, it can also be a valid dma_addr returned on some * architectures. * * The powerpc iommu assigns dma_addrs in ascending order, so we don't * have to bother with retries or mapping a dummy page to insure we * don't just get the same mapping again. * * I'm sure we won't be so lucky with other iommu's, so FIXME. */ dma_addr_t ipath_map_page(struct pci_dev *hwdev, struct page *page, unsigned long offset, size_t size, int direction) { dma_addr_t phys; phys = pci_map_page(hwdev, page, offset, size, direction); if (phys == 0) { pci_unmap_page(hwdev, phys, size, direction); phys = pci_map_page(hwdev, page, offset, size, direction); /* * FIXME: If we get 0 again, we should keep this page, * map another, then free the 0 page. */ } return phys; }
dma_addr_t dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction) { if (dev->bus == &pci_bus_type) return pci_map_page(to_pci_dev(dev), page, offset, size, (int)direction); if (dev->bus == &vio_bus_type) return vio_map_page(to_vio_dev(dev), page, offset, size, direction); BUG(); return (dma_addr_t)0; }
static int radeon_ttm_tt_populate(struct ttm_tt *ttm) { struct radeon_device *rdev; struct radeon_ttm_tt *gtt = (void *)ttm; unsigned i; int r; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); if (ttm->state != tt_unpopulated) return 0; if (slave && ttm->sg) { drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, gtt->ttm.dma_address, ttm->num_pages); ttm->state = tt_unbound; return 0; } rdev = radeon_get_rdev(ttm->bdev); #if __OS_HAS_AGP if (rdev->flags & RADEON_IS_AGP) { return ttm_agp_tt_populate(ttm); } #endif #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0)) #ifdef CONFIG_SWIOTLB if (swiotlb_nr_tbl()) { return ttm_dma_populate(>t->ttm, rdev->dev); } #endif #endif r = ttm_pool_populate(ttm); if (r) { return r; } for (i = 0; i < ttm->num_pages; i++) { gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { while (--i) { pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); gtt->ttm.dma_address[i] = 0; } ttm_pool_unpopulate(ttm); return -EFAULT; } } return 0; }
/** * efx_init_rx_buffers_page - create EFX_RX_BATCH page-based RX buffers * * @rx_queue: Efx RX queue * * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA, * and populates struct efx_rx_buffers for each one. Return a negative error * code or 0 on success. If a single page can be split between two buffers, * then the page will either be inserted fully, or not at at all. */ static int efx_init_rx_buffers_page(struct efx_rx_queue *rx_queue) { struct efx_nic *efx = rx_queue->efx; struct efx_rx_buffer *rx_buf; struct page *page; void *page_addr; struct efx_rx_page_state *state; dma_addr_t dma_addr; unsigned index, count; /* We can split a page between two buffers */ BUILD_BUG_ON(EFX_RX_BATCH & 1); for (count = 0; count < EFX_RX_BATCH; ++count) { page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC, efx->rx_buffer_order); if (unlikely(page == NULL)) return -ENOMEM; dma_addr = pci_map_page(efx->pci_dev, page, 0, efx_rx_buf_size(efx), PCI_DMA_FROMDEVICE); if (unlikely(pci_dma_mapping_error(efx->pci_dev, dma_addr))) { __free_pages(page, efx->rx_buffer_order); return -EIO; } page_addr = page_address(page); state = page_addr; state->refcnt = 0; state->dma_addr = dma_addr; page_addr += sizeof(struct efx_rx_page_state); dma_addr += sizeof(struct efx_rx_page_state); split: index = rx_queue->added_count & rx_queue->ptr_mask; rx_buf = efx_rx_buffer(rx_queue, index); rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN; rx_buf->skb = NULL; rx_buf->page = page; rx_buf->data = page_addr + EFX_PAGE_IP_ALIGN; rx_buf->len = efx->rx_buffer_len - EFX_PAGE_IP_ALIGN; ++rx_queue->added_count; ++rx_queue->alloc_page_count; ++state->refcnt; if ((~count & 1) && (efx->rx_buffer_len <= EFX_RX_HALF_PAGE)) { /* Use the second half of the page */ get_page(page); dma_addr += (PAGE_SIZE >> 1); page_addr += (PAGE_SIZE >> 1); ++count; goto split; } }
/* Map a set of buffers described by scatterlist in streaming * mode for DMA. This is the scatter-gather version of the * above pci_map_single interface. Here the scatter gather list * elements are each tagged with the appropriate dma address * and length. They are obtained via sg_dma_{address,length}(SG). * * NOTE: An implementation may be able to use a smaller number of * DMA address/length pairs than there are SG table elements. * (for example via virtual mapping capabilities) * The routine returns the number of addr/length pairs actually * used, at most nents. * * Device ownership issues as mentioned above for pci_map_single are * the same here. */ int pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction) { int i; BUG_ON(direction == PCI_DMA_NONE); for (i = 0; i < nents; i++ ) { struct scatterlist *s = &sg[i]; BUG_ON(!s->page); s->dma_address = pci_map_page(hwdev, s->page, s->offset, s->length, direction); s->dma_length = s->length; } return nents; }
/** * amdgpu_dummy_page_init - init dummy page used by the driver * * @adev: amdgpu_device pointer * * Allocate the dummy page used by the driver (all asics). * This dummy page is used by the driver as a filler for gart entries * when pages are taken out of the GART * Returns 0 on sucess, -ENOMEM on failure. */ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) { struct page *dummy_page = adev->mman.bdev.glob->dummy_read_page; if (adev->dummy_page_addr) return 0; adev->dummy_page_addr = pci_map_page(adev->pdev, dummy_page, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(adev->pdev, adev->dummy_page_addr)) { dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n"); adev->dummy_page_addr = 0; return -ENOMEM; } return 0; }
static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx, skb_frag_t *frag) { st->unmap_addr = pci_map_page(efx->pci_dev, frag->page, frag->page_offset, frag->size, PCI_DMA_TODEVICE); if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) { st->unmap_single = false; st->unmap_len = frag->size; st->in_len = frag->size; st->dma_addr = st->unmap_addr; return 0; } return -ENOMEM; }
/** * tso_get_fragment - record fragment details and map for DMA * @st: TSO state * @efx: Efx NIC * @data: Pointer to fragment data * @len: Length of fragment * * Record fragment details and map for DMA. Return 0 on success, or * -%ENOMEM if DMA mapping fails. */ static inline int tso_get_fragment(struct tso_state *st, struct efx_nic *efx, int len, struct page *page, int page_off) { st->ifc.unmap_addr = pci_map_page(efx->pci_dev, page, page_off, len, PCI_DMA_TODEVICE); if (likely(!pci_dma_mapping_error(efx->pci_dev, st->ifc.unmap_addr))) { st->ifc.unmap_len = len; st->ifc.len = len; st->ifc.dma_addr = st->ifc.unmap_addr; st->ifc.page = page; st->ifc.page_off = page_off; return 0; } return -ENOMEM; }
static inline void enic_queue_wq_skb_cont(struct enic *enic, struct vnic_wq *wq, struct sk_buff *skb, unsigned int len_left) { skb_frag_t *frag; for (frag = skb_shinfo(skb)->frags; len_left; frag++) { len_left -= frag->size; enic_queue_wq_desc_cont(wq, skb, pci_map_page(enic->pdev, frag->page, frag->page_offset, frag->size, PCI_DMA_TODEVICE), frag->size, (len_left == 0)); } }
static int radeon_ttm_tt_populate(struct ttm_tt *ttm) { struct radeon_device *rdev; struct radeon_ttm_tt *gtt = (void *)ttm; unsigned i; int r; if (ttm->state != tt_unpopulated) return 0; rdev = radeon_get_rdev(ttm->bdev); #if __OS_HAS_AGP if (rdev->flags & RADEON_IS_AGP) { return ttm_agp_tt_populate(ttm); } #endif #ifdef CONFIG_SWIOTLB if (swiotlb_nr_tbl()) { return ttm_dma_populate(>t->ttm, rdev->dev); } #endif r = ttm_pool_populate(ttm); if (r) { return r; } for (i = 0; i < ttm->num_pages; i++) { gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { while (--i) { pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); gtt->ttm.dma_address[i] = 0; } ttm_pool_unpopulate(ttm); return -EFAULT; } } return 0; }
struct dl_dma_list* dl_dma_map_kernel_buffer(void *address, unsigned long size, int direction, int is_vmalloc, void* pdev) { struct page* page; int i = 0, offset = 0; struct dl_dma_list* sl = NULL; struct dl_dma_entry *e = NULL; unsigned long num_pages = dl_dma_get_num_pages(address, size); unsigned long start_addr = (unsigned long)address; start_addr = start_addr - (start_addr % PAGE_SIZE); sl = alloc_dl_dma_entry(is_vmalloc == 1 ? num_pages : 1); if (!sl) return NULL; e = first_entry(sl); direction = bmd_to_linux_direction(direction); if (is_vmalloc) { for (i = 0; i < num_pages; i++) { page = vmalloc_to_page((void*)(unsigned long)start_addr + offset); offset += PAGE_SIZE; e->dma_addr = pci_map_page(pdev, page, 0, PAGE_SIZE, direction); e = next_entry(e); } sl->num_pages = num_pages; } else { e->dma_addr = pci_map_single(pdev, address, size, direction); sl->dma_is_single = 1; sl->size = size; } sl->pdev = pdev; return sl; }
/** * efx_init_rx_buffer_page - create new RX buffer using page-based allocation * * @rx_queue: Efx RX queue * @rx_buf: RX buffer structure to populate * * This allocates memory for a new receive buffer, maps it for DMA, * and populates a struct efx_rx_buffer with the relevant * information. Return a negative error code or 0 on success. */ static int efx_init_rx_buffer_page(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf) { struct efx_nic *efx = rx_queue->efx; int bytes, space, offset; bytes = efx->rx_buffer_len - EFX_PAGE_IP_ALIGN; /* If there is space left in the previously allocated page, * then use it. Otherwise allocate a new one */ rx_buf->page = rx_queue->buf_page; if (rx_buf->page == NULL) { dma_addr_t dma_addr; rx_buf->page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC, efx->rx_buffer_order); if (unlikely(rx_buf->page == NULL)) return -ENOMEM; dma_addr = pci_map_page(efx->pci_dev, rx_buf->page, 0, efx_rx_buf_size(efx), PCI_DMA_FROMDEVICE); if (unlikely(pci_dma_mapping_error(efx->pci_dev, dma_addr))) { __free_pages(rx_buf->page, efx->rx_buffer_order); rx_buf->page = NULL; return -EIO; } rx_queue->buf_page = rx_buf->page; rx_queue->buf_dma_addr = dma_addr; rx_queue->buf_data = (page_address(rx_buf->page) + EFX_PAGE_IP_ALIGN); } rx_buf->len = bytes; rx_buf->data = rx_queue->buf_data; offset = efx_rx_buf_offset(rx_buf); rx_buf->dma_addr = rx_queue->buf_dma_addr + offset; /* Try to pack multiple buffers per page */ if (efx->rx_buffer_order == 0) { /* The next buffer starts on the next 512 byte boundary */ rx_queue->buf_data += ((bytes + 0x1ff) & ~0x1ff); offset += ((bytes + 0x1ff) & ~0x1ff); space = efx_rx_buf_size(efx) - offset; if (space >= bytes) { /* Refs dropped on kernel releasing each skb */ get_page(rx_queue->buf_page); goto out; } } /* This is the final RX buffer for this page, so mark it for * unmapping */ rx_queue->buf_page = NULL; rx_buf->unmap_addr = rx_queue->buf_dma_addr; out: return 0; }
/* * Add a socket buffer to a TX queue * * This maps all fragments of a socket buffer for DMA and adds them to * the TX queue. The queue's insert pointer will be incremented by * the number of fragments in the socket buffer. * * If any DMA mapping fails, any mapped fragments will be unmapped, * the queue's insert pointer will be restored to its original value. * * This function is split out from efx_hard_start_xmit to allow the * loopback test to direct packets via specific TX queues. * * Returns NETDEV_TX_OK or NETDEV_TX_BUSY * You must hold netif_tx_lock() to call this function. */ netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) { struct efx_nic *efx = tx_queue->efx; struct pci_dev *pci_dev = efx->pci_dev; struct efx_tx_buffer *buffer; skb_frag_t *fragment; struct page *page; int page_offset; unsigned int len, unmap_len = 0, fill_level, insert_ptr; dma_addr_t dma_addr, unmap_addr = 0; unsigned int dma_len; bool unmap_single; int q_space, i = 0; netdev_tx_t rc = NETDEV_TX_OK; EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count); if (skb_shinfo(skb)->gso_size) return efx_enqueue_skb_tso(tx_queue, skb); /* Get size of the initial fragment */ len = skb_headlen(skb); /* Pad if necessary */ if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) { EFX_BUG_ON_PARANOID(skb->data_len); len = 32 + 1; if (skb_pad(skb, len - skb->len)) return NETDEV_TX_OK; } fill_level = tx_queue->insert_count - tx_queue->old_read_count; q_space = efx->txq_entries - 1 - fill_level; /* Map for DMA. Use pci_map_single rather than pci_map_page * since this is more efficient on machines with sparse * memory. */ unmap_single = true; dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE); /* Process all fragments */ while (1) { if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr))) goto pci_err; /* Store fields for marking in the per-fragment final * descriptor */ unmap_len = len; unmap_addr = dma_addr; /* Add to TX queue, splitting across DMA boundaries */ do { if (unlikely(q_space-- <= 0)) { /* It might be that completions have * happened since the xmit path last * checked. Update the xmit path's * copy of read_count. */ netif_tx_stop_queue(tx_queue->core_txq); /* This memory barrier protects the * change of queue state from the access * of read_count. */ smp_mb(); tx_queue->old_read_count = ACCESS_ONCE(tx_queue->read_count); fill_level = (tx_queue->insert_count - tx_queue->old_read_count); q_space = efx->txq_entries - 1 - fill_level; if (unlikely(q_space-- <= 0)) { rc = NETDEV_TX_BUSY; goto unwind; } smp_mb(); netif_tx_start_queue(tx_queue->core_txq); } insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask; buffer = &tx_queue->buffer[insert_ptr]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->tsoh); EFX_BUG_ON_PARANOID(buffer->skb); EFX_BUG_ON_PARANOID(buffer->len); EFX_BUG_ON_PARANOID(!buffer->continuation); EFX_BUG_ON_PARANOID(buffer->unmap_len); dma_len = efx_max_tx_len(efx, dma_addr); if (likely(dma_len >= len)) dma_len = len; /* Fill out per descriptor fields */ buffer->len = dma_len; buffer->dma_addr = dma_addr; len -= dma_len; dma_addr += dma_len; ++tx_queue->insert_count; } while (len); /* Transfer ownership of the unmapping to the final buffer */ buffer->unmap_single = unmap_single; buffer->unmap_len = unmap_len; unmap_len = 0; /* Get address and size of next fragment */ if (i >= skb_shinfo(skb)->nr_frags) break; fragment = &skb_shinfo(skb)->frags[i]; len = fragment->size; page = fragment->page; page_offset = fragment->page_offset; i++; /* Map for DMA */ unmap_single = false; dma_addr = pci_map_page(pci_dev, page, page_offset, len, PCI_DMA_TODEVICE); } /* Transfer ownership of the skb to the final buffer */ buffer->skb = skb; buffer->continuation = false; /* Pass off to hardware */ efx_nic_push_buffers(tx_queue); return NETDEV_TX_OK; pci_err: netif_err(efx, tx_err, efx->net_dev, " TX queue %d could not map skb with %d bytes %d " "fragments for DMA\n", tx_queue->queue, skb->len, skb_shinfo(skb)->nr_frags + 1); /* Mark the packet as transmitted, and free the SKB ourselves */ dev_kfree_skb_any(skb); unwind: /* Work backwards until we hit the original insert pointer value */ while (tx_queue->insert_count != tx_queue->write_count) { --tx_queue->insert_count; insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask; buffer = &tx_queue->buffer[insert_ptr]; efx_dequeue_buffer(tx_queue, buffer); buffer->len = 0; } /* Free the fragment we were mid-way through pushing */ if (unmap_len) { if (unmap_single) pci_unmap_single(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); else pci_unmap_page(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); } return rc; }
/** * qla2x00_build_scsi_iocbs_64() - Build IOCB command utilizing 64bit * capable IOCB types. * * @sp: SRB command to process * @cmd_pkt: Command type 3 IOCB * @tot_dsds: Total number of segments to transfer */ void qla2x00_build_scsi_iocbs_64(srb_t *sp, cmd_entry_t *cmd_pkt, uint16_t tot_dsds) { uint16_t avail_dsds; uint32_t *cur_dsd; scsi_qla_host_t *ha; struct scsi_cmnd *cmd; cmd = sp->cmd; /* Update entry type to indicate Command Type 3 IOCB */ *((uint32_t *)(&cmd_pkt->entry_type)) = __constant_cpu_to_le32(COMMAND_A64_TYPE); /* No data transfer */ if (cmd->request_bufflen == 0 || cmd->sc_data_direction == DMA_NONE) { cmd_pkt->byte_count = __constant_cpu_to_le32(0); return; } ha = sp->ha; cmd_pkt->control_flags |= cpu_to_le16(qla2x00_get_cmd_direction(cmd)); /* Two DSDs are available in the Command Type 3 IOCB */ avail_dsds = 2; cur_dsd = (uint32_t *)&cmd_pkt->dseg_0_address; /* Load data segments */ if (cmd->use_sg != 0) { struct scatterlist *cur_seg; struct scatterlist *end_seg; cur_seg = (struct scatterlist *)cmd->request_buffer; end_seg = cur_seg + tot_dsds; while (cur_seg < end_seg) { dma_addr_t sle_dma; cont_a64_entry_t *cont_pkt; /* Allocate additional continuation packets? */ if (avail_dsds == 0) { /* * Five DSDs are available in the Continuation * Type 1 IOCB. */ cont_pkt = qla2x00_prep_cont_type1_iocb(ha); cur_dsd = (uint32_t *)cont_pkt->dseg_0_address; avail_dsds = 5; } sle_dma = sg_dma_address(cur_seg); *cur_dsd++ = cpu_to_le32(LSD(sle_dma)); *cur_dsd++ = cpu_to_le32(MSD(sle_dma)); *cur_dsd++ = cpu_to_le32(sg_dma_len(cur_seg)); avail_dsds--; cur_seg++; } } else { dma_addr_t req_dma; struct page *page; unsigned long offset; page = virt_to_page(cmd->request_buffer); offset = ((unsigned long)cmd->request_buffer & ~PAGE_MASK); req_dma = pci_map_page(ha->pdev, page, offset, cmd->request_bufflen, cmd->sc_data_direction); sp->dma_handle = req_dma; *cur_dsd++ = cpu_to_le32(LSD(req_dma)); *cur_dsd++ = cpu_to_le32(MSD(req_dma)); *cur_dsd++ = cpu_to_le32(cmd->request_bufflen); } }
/* * Prepare a socket for DDP. Must be called when the socket is known to be * open. */ int t3_enter_ddp(struct toepcb *toep, unsigned int kbuf_size, unsigned int waitall, int nonblock) { int i, err = ENOMEM; static vm_pindex_t color; unsigned int nppods, kbuf_pages, idx = 0; struct ddp_state *p = &toep->tp_ddp_state; struct tom_data *d = TOM_DATA(toep->tp_toedev); if (kbuf_size > M_TCB_RX_DDP_BUF0_LEN) return (EINVAL); #ifdef notyet SOCKBUF_LOCK_ASSERT(&so->so_rcv); #endif kbuf_pages = (kbuf_size + PAGE_SIZE - 1) >> PAGE_SHIFT; nppods = pages2ppods(kbuf_pages); p->kbuf_noinval = !!waitall; p->kbuf_tag[NUM_DDP_KBUF - 1] = -1; for (idx = 0; idx < NUM_DDP_KBUF; idx++) { p->kbuf[idx] = malloc(sizeof (struct ddp_gather_list) + kbuf_pages * sizeof(vm_page_t *), M_DEVBUF, M_NOWAIT|M_ZERO); if (p->kbuf[idx] == NULL) goto err; err = t3_alloc_ppods(d, nppods, &p->kbuf_tag[idx]); if (err) { printf("t3_alloc_ppods failed err=%d\n", err); goto err; } p->kbuf_nppods[idx] = nppods; p->kbuf[idx]->dgl_length = kbuf_size; p->kbuf[idx]->dgl_offset = 0; p->kbuf[idx]->dgl_nelem = kbuf_pages; for (i = 0; i < kbuf_pages; ++i) { p->kbuf[idx]->dgl_pages[i] = vm_page_alloc(NULL, color, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (p->kbuf[idx]->dgl_pages[i] == NULL) { p->kbuf[idx]->dgl_nelem = i; printf("failed to allocate kbuf pages\n"); goto err; } } #ifdef NEED_BUSDMA /* * XXX we'll need this for VT-d or any platform with an iommu :-/ * */ for (i = 0; i < kbuf_pages; ++i) p->kbuf[idx]->phys_addr[i] = pci_map_page(p->pdev, p->kbuf[idx]->pages[i], 0, PAGE_SIZE, PCI_DMA_FROMDEVICE); #endif t3_setup_ppods(toep, p->kbuf[idx], nppods, p->kbuf_tag[idx], p->kbuf[idx]->dgl_length, 0, 0); } cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid); t3_set_ddp_tag(toep, 0, p->kbuf_tag[0] << 6); t3_set_ddp_buf(toep, 0, 0, p->kbuf[0]->dgl_length); t3_repost_kbuf(toep, 0, 0, 1, nonblock); t3_set_rcv_coalesce_enable(toep, TOM_TUNABLE(toep->tp_toedev, ddp_rcvcoalesce)); t3_set_dack_mss(toep, TOM_TUNABLE(toep->tp_toedev, delack)>>1); #ifdef T3_TRACE T3_TRACE4(TIDTB(so), "t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d", kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]); #endif CTR4(KTR_TOM, "t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d", kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]); cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid); return (0); err: t3_release_ddp_resources(toep); t3_cleanup_ddp(toep); return (err); }
/* * Add a socket buffer to a TX queue * * This maps all fragments of a socket buffer for DMA and adds them to * the TX queue. The queue's insert pointer will be incremented by * the number of fragments in the socket buffer. * * If any DMA mapping fails, any mapped fragments will be unmapped, * the queue's insert pointer will be restored to its original value. * * Returns NETDEV_TX_OK or NETDEV_TX_BUSY * You must hold netif_tx_lock() to call this function. */ static inline int efx_enqueue_skb(struct efx_tx_queue *tx_queue, const struct sk_buff *skb) { struct efx_nic *efx = tx_queue->efx; struct pci_dev *pci_dev = efx->pci_dev; struct efx_tx_buffer *buffer; skb_frag_t *fragment; struct page *page; int page_offset; unsigned int len, unmap_len = 0, fill_level, insert_ptr, misalign; dma_addr_t dma_addr, unmap_addr = 0; unsigned int dma_len; unsigned unmap_single; int q_space, i = 0; int rc = NETDEV_TX_OK; EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count); if (skb_shinfo((struct sk_buff *)skb)->gso_size) return efx_enqueue_skb_tso(tx_queue, skb); /* Get size of the initial fragment */ len = skb_headlen(skb); fill_level = tx_queue->insert_count - tx_queue->old_read_count; q_space = efx->type->txd_ring_mask - 1 - fill_level; /* Map for DMA. Use pci_map_single rather than pci_map_page * since this is more efficient on machines with sparse * memory. */ unmap_single = 1; dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE); /* Process all fragments */ while (1) { if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr))) goto pci_err; /* Store fields for marking in the per-fragment final * descriptor */ unmap_len = len; unmap_addr = dma_addr; /* Add to TX queue, splitting across DMA boundaries */ do { if (unlikely(q_space-- <= 0)) { /* It might be that completions have * happened since the xmit path last * checked. Update the xmit path's * copy of read_count. */ ++tx_queue->stopped; /* This memory barrier protects the * change of stopped from the access * of read_count. */ smp_mb(); tx_queue->old_read_count = *(volatile unsigned *) &tx_queue->read_count; fill_level = (tx_queue->insert_count - tx_queue->old_read_count); q_space = (efx->type->txd_ring_mask - 1 - fill_level); if (unlikely(q_space-- <= 0)) goto stop; smp_mb(); --tx_queue->stopped; } insert_ptr = (tx_queue->insert_count & efx->type->txd_ring_mask); buffer = &tx_queue->buffer[insert_ptr]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->tsoh); EFX_BUG_ON_PARANOID(buffer->skb); EFX_BUG_ON_PARANOID(buffer->len); EFX_BUG_ON_PARANOID(buffer->continuation != 1); EFX_BUG_ON_PARANOID(buffer->unmap_len); dma_len = (((~dma_addr) & efx->type->tx_dma_mask) + 1); if (likely(dma_len > len)) dma_len = len; misalign = (unsigned)dma_addr & efx->type->bug5391_mask; if (misalign && dma_len + misalign > 512) dma_len = 512 - misalign; /* Fill out per descriptor fields */ buffer->len = dma_len; buffer->dma_addr = dma_addr; len -= dma_len; dma_addr += dma_len; ++tx_queue->insert_count; } while (len); /* Transfer ownership of the unmapping to the final buffer */ buffer->unmap_addr = unmap_addr; buffer->unmap_single = unmap_single; buffer->unmap_len = unmap_len; unmap_len = 0; /* Get address and size of next fragment */ if (i >= skb_shinfo(skb)->nr_frags) break; fragment = &skb_shinfo(skb)->frags[i]; len = fragment->size; page = fragment->page; page_offset = fragment->page_offset; i++; /* Map for DMA */ unmap_single = 0; dma_addr = pci_map_page(pci_dev, page, page_offset, len, PCI_DMA_TODEVICE); } /* Transfer ownership of the skb to the final buffer */ buffer->skb = skb; buffer->continuation = 0; /* Pass off to hardware */ falcon_push_buffers(tx_queue); return NETDEV_TX_OK; pci_err: EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d " "fragments for DMA\n", tx_queue->queue, skb->len, skb_shinfo(skb)->nr_frags + 1); /* Mark the packet as transmitted, and free the SKB ourselves */ dev_kfree_skb_any((struct sk_buff *)skb); goto unwind; stop: rc = NETDEV_TX_BUSY; if (tx_queue->stopped == 1) efx_stop_queue(efx); unwind: /* Work backwards until we hit the original insert pointer value */ while (tx_queue->insert_count != tx_queue->write_count) { --tx_queue->insert_count; insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask; buffer = &tx_queue->buffer[insert_ptr]; efx_dequeue_buffer(tx_queue, buffer); buffer->len = 0; } /* Free the fragment we were mid-way through pushing */ if (unmap_len) pci_unmap_page(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); return rc; }
static int c2_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct c2_port *c2_port = netdev_priv(netdev); struct c2_dev *c2dev = c2_port->c2dev; struct c2_ring *tx_ring = &c2_port->tx_ring; struct c2_element *elem; dma_addr_t mapaddr; u32 maplen; unsigned long flags; unsigned int i; spin_lock_irqsave(&c2_port->tx_lock, flags); if (unlikely(c2_port->tx_avail < (skb_shinfo(skb)->nr_frags + 1))) { netif_stop_queue(netdev); spin_unlock_irqrestore(&c2_port->tx_lock, flags); pr_debug("%s: Tx ring full when queue awake!\n", netdev->name); return NETDEV_TX_BUSY; } maplen = skb_headlen(skb); mapaddr = pci_map_single(c2dev->pcidev, skb->data, maplen, PCI_DMA_TODEVICE); elem = tx_ring->to_use; elem->skb = skb; elem->mapaddr = mapaddr; elem->maplen = maplen; /* Tell HW to xmit */ __raw_writeq((__force u64) cpu_to_be64(mapaddr), elem->hw_desc + C2_TXP_ADDR); __raw_writew((__force u16) cpu_to_be16(maplen), elem->hw_desc + C2_TXP_LEN); __raw_writew((__force u16) cpu_to_be16(TXP_HTXD_READY), elem->hw_desc + C2_TXP_FLAGS); netdev->stats.tx_packets++; netdev->stats.tx_bytes += maplen; /* Loop thru additional data fragments and queue them */ if (skb_shinfo(skb)->nr_frags) { for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; maplen = frag->size; mapaddr = pci_map_page(c2dev->pcidev, frag->page, frag->page_offset, maplen, PCI_DMA_TODEVICE); elem = elem->next; elem->skb = NULL; elem->mapaddr = mapaddr; elem->maplen = maplen; /* Tell HW to xmit */ __raw_writeq((__force u64) cpu_to_be64(mapaddr), elem->hw_desc + C2_TXP_ADDR); __raw_writew((__force u16) cpu_to_be16(maplen), elem->hw_desc + C2_TXP_LEN); __raw_writew((__force u16) cpu_to_be16(TXP_HTXD_READY), elem->hw_desc + C2_TXP_FLAGS); netdev->stats.tx_packets++; netdev->stats.tx_bytes += maplen; } } tx_ring->to_use = elem->next; c2_port->tx_avail -= (skb_shinfo(skb)->nr_frags + 1); if (c2_port->tx_avail <= MAX_SKB_FRAGS + 1) { netif_stop_queue(netdev); if (netif_msg_tx_queued(c2_port)) pr_debug("%s: transmit queue full\n", netdev->name); } spin_unlock_irqrestore(&c2_port->tx_lock, flags); netdev->trans_start = jiffies; return NETDEV_TX_OK; }
static int radeon_ttm_tt_populate(struct ttm_tt *ttm) { struct radeon_device *rdev; struct radeon_ttm_tt *gtt = (void *)ttm; unsigned i; int r; #ifdef DUMBBELL_WIP bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); #endif /* DUMBBELL_WIP */ if (ttm->state != tt_unpopulated) return 0; #ifdef DUMBBELL_WIP /* * Maybe unneeded on FreeBSD. * -- dumbbell@ */ if (slave && ttm->sg) { drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, gtt->ttm.dma_address, ttm->num_pages); ttm->state = tt_unbound; return 0; } #endif /* DUMBBELL_WIP */ rdev = radeon_get_rdev(ttm->bdev); #if __OS_HAS_AGP #ifdef DUMBBELL_WIP if (rdev->flags & RADEON_IS_AGP) { return ttm_agp_tt_populate(ttm); } #endif /* DUMBBELL_WIP */ #endif #ifdef CONFIG_SWIOTLB if (swiotlb_nr_tbl()) { return ttm_dma_populate(>t->ttm, rdev->dev); } #endif r = ttm_pool_populate(ttm); if (r) { return r; } for (i = 0; i < ttm->num_pages; i++) { gtt->ttm.dma_address[i] = VM_PAGE_TO_PHYS(ttm->pages[i]); #ifdef DUMBBELL_WIP gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { while (--i) { pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); gtt->ttm.dma_address[i] = 0; } ttm_pool_unpopulate(ttm); return -EFAULT; } #endif /* DUMBBELL_WIP */ } return 0; }
static inline void enic_queue_wq_skb_tso(struct enic *enic, struct vnic_wq *wq, struct sk_buff *skb, unsigned int mss, int vlan_tag_insert, unsigned int vlan_tag) { unsigned int frag_len_left = skb_headlen(skb); unsigned int len_left = skb->len - frag_len_left; unsigned int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); int eop = (len_left == 0); unsigned int len; dma_addr_t dma_addr; unsigned int offset = 0; skb_frag_t *frag; if (skb->protocol == cpu_to_be16(ETH_P_IP)) { ip_hdr(skb)->check = 0; tcp_hdr(skb)->check = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); } while (frag_len_left) { len = min(frag_len_left, (unsigned int)WQ_ENET_MAX_DESC_LEN); dma_addr = pci_map_single(enic->pdev, skb->data + offset, len, PCI_DMA_TODEVICE); enic_queue_wq_desc_tso(wq, skb, dma_addr, len, mss, hdr_len, vlan_tag_insert, vlan_tag, eop && (len == frag_len_left)); frag_len_left -= len; offset += len; } if (eop) return; for (frag = skb_shinfo(skb)->frags; len_left; frag++) { len_left -= frag->size; frag_len_left = frag->size; offset = frag->page_offset; while (frag_len_left) { len = min(frag_len_left, (unsigned int)WQ_ENET_MAX_DESC_LEN); dma_addr = pci_map_page(enic->pdev, frag->page, offset, len, PCI_DMA_TODEVICE); enic_queue_wq_desc_cont(wq, skb, dma_addr, len, (len_left == 0) && (len == frag_len_left)); frag_len_left -= len; offset += len; } } }
/** * iwl_rx_replenish - Move all used packet from rx_used to rx_free * * When moving to rx_free an SKB is allocated for the slot. * * Also restock the Rx queue via iwl_rx_queue_restock. * This is called as a scheduled work item (except for during initialization) */ void iwl_rx_allocate(struct iwl_priv *priv, gfp_t priority) { struct iwl_rx_queue *rxq = &priv->rxq; struct list_head *element; struct iwl_rx_mem_buffer *rxb; struct page *page; unsigned long flags; gfp_t gfp_mask = priority; while (1) { spin_lock_irqsave(&rxq->lock, flags); if (list_empty(&rxq->rx_used)) { spin_unlock_irqrestore(&rxq->lock, flags); return; } spin_unlock_irqrestore(&rxq->lock, flags); if (rxq->free_count > RX_LOW_WATERMARK) gfp_mask |= __GFP_NOWARN; if (priv->hw_params.rx_page_order > 0) gfp_mask |= __GFP_COMP; /* Alloc a new receive buffer */ page = alloc_pages(gfp_mask, priv->hw_params.rx_page_order); if (!page) { if (net_ratelimit()) IWL_DEBUG_INFO(priv, "alloc_pages failed, " "order: %d\n", priv->hw_params.rx_page_order); if ((rxq->free_count <= RX_LOW_WATERMARK) && net_ratelimit()) IWL_CRIT(priv, "Failed to alloc_pages with %s. Only %u free buffers remaining.\n", priority == GFP_ATOMIC ? "GFP_ATOMIC" : "GFP_KERNEL", rxq->free_count); /* We don't reschedule replenish work here -- we will * call the restock method and if it still needs * more buffers it will schedule replenish */ return; } spin_lock_irqsave(&rxq->lock, flags); if (list_empty(&rxq->rx_used)) { spin_unlock_irqrestore(&rxq->lock, flags); __free_pages(page, priv->hw_params.rx_page_order); return; } element = rxq->rx_used.next; rxb = list_entry(element, struct iwl_rx_mem_buffer, list); list_del(element); spin_unlock_irqrestore(&rxq->lock, flags); rxb->page = page; /* Get physical address of the RB */ rxb->page_dma = pci_map_page(priv->pci_dev, page, 0, PAGE_SIZE << priv->hw_params.rx_page_order, PCI_DMA_FROMDEVICE); /* dma address must be no more than 36 bits */ BUG_ON(rxb->page_dma & ~DMA_BIT_MASK(36)); /* and also 256 byte aligned! */ BUG_ON(rxb->page_dma & DMA_BIT_MASK(8)); spin_lock_irqsave(&rxq->lock, flags); list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; priv->alloc_rxb_page++; spin_unlock_irqrestore(&rxq->lock, flags); } }