/** * iwl_cmd_queue_unmap - Unmap any remaining DMA mappings from command queue */ void iwl_legacy_cmd_queue_unmap(struct iwl_priv *priv) { struct iwl_tx_queue *txq = &priv->txq[priv->cmd_queue]; struct iwl_queue *q = &txq->q; int i; if (q->n_bd == 0) return; while (q->read_ptr != q->write_ptr) { i = iwl_legacy_get_cmd_index(q, q->read_ptr, 0); if (txq->meta[i].flags & CMD_MAPPED) { pci_unmap_single(priv->pci_dev, dma_unmap_addr(&txq->meta[i], mapping), dma_unmap_len(&txq->meta[i], len), PCI_DMA_BIDIRECTIONAL); txq->meta[i].flags = 0; } q->read_ptr = iwl_legacy_queue_inc_wrap(q->read_ptr, q->n_bd); } i = q->n_window; if (txq->meta[i].flags & CMD_MAPPED) { pci_unmap_single(priv->pci_dev, dma_unmap_addr(&txq->meta[i], mapping), dma_unmap_len(&txq->meta[i], len), PCI_DMA_BIDIRECTIONAL); txq->meta[i].flags = 0; } }
static void iwlagn_unmap_tfd(struct iwl_priv *priv, struct iwl_cmd_meta *meta, struct iwl_tfd *tfd, enum dma_data_direction dma_dir) { int i; int num_tbs; /* Sanity check on number of chunks */ num_tbs = iwl_tfd_get_num_tbs(tfd); if (num_tbs >= IWL_NUM_OF_TBS) { IWL_ERR(priv, "Too many chunks: %i\n", num_tbs); /* @todo issue fatal error, it is quite serious situation */ return; } /* Unmap tx_cmd */ if (num_tbs) dma_unmap_single(priv->bus->dev, dma_unmap_addr(meta, mapping), dma_unmap_len(meta, len), DMA_BIDIRECTIONAL); /* Unmap chunks, if any. */ for (i = 1; i < num_tbs; i++) dma_unmap_single(priv->bus->dev, iwl_tfd_tb_get_addr(tfd, i), iwl_tfd_tb_get_len(tfd, i), dma_dir); }
static void iwlagn_unmap_tfd(struct iwl_trans *trans, struct iwl_cmd_meta *meta, struct iwl_tfd *tfd, enum dma_data_direction dma_dir) { int i; int num_tbs; num_tbs = iwl_tfd_get_num_tbs(tfd); if (num_tbs >= IWL_NUM_OF_TBS) { IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); return; } if (num_tbs) dma_unmap_single(trans->dev, dma_unmap_addr(meta, mapping), dma_unmap_len(meta, len), DMA_BIDIRECTIONAL); for (i = 1; i < num_tbs; i++) dma_unmap_single(trans->dev, iwl_tfd_tb_get_addr(tfd, i), iwl_tfd_tb_get_len(tfd, i), dma_dir); }
static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx, struct sk_buff *skb, struct c4iw_wr_wait *wr_waitp) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; int ret; wr_len = sizeof *res_wr + sizeof *res; set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = __skb_put_zero(skb, wr_len); res_wr->op_nres = cpu_to_be32( FW_WR_OP_V(FW_RI_RES_WR) | FW_RI_RES_WR_NRES_V(1) | FW_WR_COMPL_F); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (uintptr_t)wr_waitp; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_RESET; res->u.cq.iqid = cpu_to_be32(cq->cqid); c4iw_init_wr_wait(wr_waitp); ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__); kfree(cq->sw_queue); dma_free_coherent(&(rdev->lldi.pdev->dev), cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); return ret; }
/** * iwl_legacy_tx_cmd_complete - Pull unused buffers off the queue and reclaim them * @rxb: Rx buffer to reclaim * * If an Rx buffer has an async callback associated with it the callback * will be executed. The attached skb (if present) will only be freed * if the callback returns 1 */ void iwl_legacy_tx_cmd_complete(struct iwl_priv *priv, struct iwl_rx_mem_buffer *rxb) { struct iwl_rx_packet *pkt = rxb_addr(rxb); u16 sequence = le16_to_cpu(pkt->hdr.sequence); int txq_id = SEQ_TO_QUEUE(sequence); int index = SEQ_TO_INDEX(sequence); int cmd_index; bool huge = !!(pkt->hdr.sequence & SEQ_HUGE_FRAME); struct iwl_device_cmd *cmd; struct iwl_cmd_meta *meta; struct iwl_tx_queue *txq = &priv->txq[priv->cmd_queue]; unsigned long flags; /* If a Tx command is being handled and it isn't in the actual * command queue then there a command routing bug has been introduced * in the queue management code. */ if (WARN(txq_id != priv->cmd_queue, "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n", txq_id, priv->cmd_queue, sequence, priv->txq[priv->cmd_queue].q.read_ptr, priv->txq[priv->cmd_queue].q.write_ptr)) { iwl_print_hex_error(priv, pkt, 32); return; } cmd_index = iwl_legacy_get_cmd_index(&txq->q, index, huge); cmd = txq->cmd[cmd_index]; meta = &txq->meta[cmd_index]; txq->time_stamp = jiffies; pci_unmap_single(priv->pci_dev, dma_unmap_addr(meta, mapping), dma_unmap_len(meta, len), PCI_DMA_BIDIRECTIONAL); /* Input error checking is done when commands are added to queue. */ if (meta->flags & CMD_WANT_SKB) { meta->source->reply_page = (unsigned long)rxb_addr(rxb); rxb->page = NULL; } else if (meta->callback) meta->callback(priv, cmd, pkt); spin_lock_irqsave(&priv->hcmd_lock, flags); iwl_legacy_hcmd_queue_reclaim(priv, txq_id, index, cmd_index); if (!(meta->flags & CMD_ASYNC)) { clear_bit(STATUS_HCMD_ACTIVE, &priv->status); IWL_DEBUG_INFO(priv, "Clearing HCMD_ACTIVE for command %s\n", iwl_legacy_get_cmd_string(cmd->hdr.cmd)); wake_up(&priv->wait_command_queue); } /* Mark as unmapped */ meta->flags = 0; spin_unlock_irqrestore(&priv->hcmd_lock, flags); }
static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *tx_bi) { xdp_return_frame(tx_bi->xdpf); dma_unmap_single(tx_ring->dev, dma_unmap_addr(tx_bi, dma), dma_unmap_len(tx_bi, len), DMA_TO_DEVICE); dma_unmap_len_set(tx_bi, len, 0); }
void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, int is_direct, struct mthca_mr *mr) { int i; if (mr) mthca_free_mr(dev, mr); if (is_direct) dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, dma_unmap_addr(&buf->direct, mapping)); else { for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf->page_list[i].buf, dma_unmap_addr(&buf->page_list[i], mapping)); kfree(buf->page_list); } }
void c2_free_mqsp_pool(struct c2_dev *c2dev, struct sp_chunk *root) { struct sp_chunk *next; while (root) { next = root->next; dma_free_coherent(&c2dev->pcidev->dev, PAGE_SIZE, root, dma_unmap_addr(root, mapping)); root = next; } }
/** * arc_emac_tx_clean - clears processed by EMAC Tx BDs. * @ndev: Pointer to the network device. */ static void arc_emac_tx_clean(struct net_device *ndev) { struct arc_emac_priv *priv = netdev_priv(ndev); struct net_device_stats *stats = &ndev->stats; unsigned int i; for (i = 0; i < TX_BD_NUM; i++) { unsigned int *txbd_dirty = &priv->txbd_dirty; struct arc_emac_bd *txbd = &priv->txbd[*txbd_dirty]; struct buffer_state *tx_buff = &priv->tx_buff[*txbd_dirty]; struct sk_buff *skb = tx_buff->skb; unsigned int info = le32_to_cpu(txbd->info); if ((info & FOR_EMAC) || !txbd->data || !skb) break; if (unlikely(info & (DROP | DEFR | LTCL | UFLO))) { stats->tx_errors++; stats->tx_dropped++; if (info & DEFR) stats->tx_carrier_errors++; if (info & LTCL) stats->collisions++; if (info & UFLO) stats->tx_fifo_errors++; } else if (likely(info & FIRST_OR_LAST_MASK)) { stats->tx_packets++; stats->tx_bytes += skb->len; } dma_unmap_single(&ndev->dev, dma_unmap_addr(tx_buff, addr), dma_unmap_len(tx_buff, len), DMA_TO_DEVICE); /* return the sk_buff to system */ dev_consume_skb_irq(skb); txbd->data = 0; txbd->info = 0; tx_buff->skb = NULL; *txbd_dirty = (*txbd_dirty + 1) % TX_BD_NUM; } /* Ensure that txbd_dirty is visible to tx() before checking * for queue stopped. */ smp_mb(); if (netif_queue_stopped(ndev) && arc_emac_tx_avail(priv)) netif_wake_queue(ndev); }
void __devexit c2_rnic_term(struct c2_dev *c2dev) { /* Close the open adapter instance */ c2_rnic_close(c2dev); /* Send the TERM message to the adapter */ c2_adapter_term(c2dev); /* Disable interrupts on the adapter */ writel(1, c2dev->regs + C2_IDIS); /* Free the QP pool */ c2_cleanup_qp_table(c2dev); /* Free the PD pool */ c2_cleanup_pd_table(c2dev); /* Free the verbs request allocator */ vq_term(c2dev); /* Free the asynchronus event queue */ dma_free_coherent(&c2dev->pcidev->dev, c2dev->aeq.q_size * c2dev->aeq.msg_size, c2dev->aeq.msg_pool.host, dma_unmap_addr(&c2dev->aeq, mapping)); /* Free the verbs reply queue */ dma_free_coherent(&c2dev->pcidev->dev, c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size, c2dev->rep_vq.msg_pool.host, dma_unmap_addr(&c2dev->rep_vq, mapping)); /* Free the MQ shared pointer pool */ c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool); /* Free the qptr_array */ vfree(c2dev->qptr_array); return; }
void fm10k_unmap_and_free_tx_resource(struct fm10k_ring *ring, struct fm10k_tx_buffer *tx_buffer) { if (tx_buffer->skb) { dev_kfree_skb_any(tx_buffer->skb); if (dma_unmap_len(tx_buffer, len)) dma_unmap_single(ring->dev, dma_unmap_addr(tx_buffer, dma), dma_unmap_len(tx_buffer, len), DMA_TO_DEVICE); } else if (dma_unmap_len(tx_buffer, len)) { dma_unmap_page(ring->dev, dma_unmap_addr(tx_buffer, dma), dma_unmap_len(tx_buffer, len), DMA_TO_DEVICE); } tx_buffer->next_to_watch = NULL; tx_buffer->skb = NULL; dma_unmap_len_set(tx_buffer, len, 0); /* tx_buffer must be completely set up in the transmit path */ }
static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; struct c4iw_wr_wait wr_wait; struct sk_buff *skb; int ret; wr_len = sizeof *res_wr + sizeof *res; skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); if (!skb) return -ENOMEM; set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = (struct fw_ri_res_wr *)__skb_put(skb, wr_len); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(1) | FW_WR_COMPL(1)); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (u64)&wr_wait; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_RESET; res->u.cq.iqid = cpu_to_be32(cq->cqid); c4iw_init_wr_wait(&wr_wait); ret = c4iw_ofld_send(rdev, skb); if (!ret) { wait_event_timeout(wr_wait.wait, wr_wait.done, C4IW_WR_TO); if (!wr_wait.done) { printk(KERN_ERR MOD "Device %s not responding!\n", pci_name(rdev->lldi.pdev)); rdev->flags = T4_FATAL_ERROR; ret = -EIO; } else ret = wr_wait.ret; } kfree(cq->sw_queue); dma_free_coherent(&(rdev->lldi.pdev->dev), cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); return ret; }
static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, struct c4iw_dev_ucontext *uctx) { /* * uP clears EQ contexts when the connection exits rdma mode, * so no need to post a RESET WR for these EQs. */ dma_free_coherent(&(rdev->lldi.pdev->dev), wq->rq.memsize, wq->rq.queue, dma_unmap_addr(&wq->rq, mapping)); dealloc_sq(rdev, &wq->sq); c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); kfree(wq->rq.sw_rq); kfree(wq->sq.sw_sq); c4iw_put_qpid(rdev, wq->rq.qid, uctx); c4iw_put_qpid(rdev, wq->sq.qid, uctx); return 0; }
/** * il3945_hw_txq_free_tfd - Free one TFD, those at idx [txq->q.read_ptr] * * Does NOT advance any idxes */ void il3945_hw_txq_free_tfd(struct il_priv *il, struct il_tx_queue *txq) { struct il3945_tfd *tfd_tmp = (struct il3945_tfd *)txq->tfds; int idx = txq->q.read_ptr; struct il3945_tfd *tfd = &tfd_tmp[idx]; struct pci_dev *dev = il->pci_dev; int i; int counter; /* sanity check */ counter = TFD_CTL_COUNT_GET(le32_to_cpu(tfd->control_flags)); if (counter > NUM_TFD_CHUNKS) { IL_ERR("Too many chunks: %i\n", counter); /* @todo issue fatal error, it is quite serious situation */ return; } /* Unmap tx_cmd */ if (counter) pci_unmap_single(dev, dma_unmap_addr(&txq->meta[idx], mapping), dma_unmap_len(&txq->meta[idx], len), PCI_DMA_TODEVICE); /* unmap chunks if any */ for (i = 1; i < counter; i++) pci_unmap_single(dev, le32_to_cpu(tfd->tbs[i].addr), le32_to_cpu(tfd->tbs[i].len), PCI_DMA_TODEVICE); /* free SKB */ if (txq->skbs) { struct sk_buff *skb = txq->skbs[txq->q.read_ptr]; /* can be called from irqs-disabled context */ if (skb) { dev_kfree_skb_any(skb); txq->skbs[txq->q.read_ptr] = NULL; } } }
static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; struct c4iw_wr_wait wr_wait; struct sk_buff *skb; int ret; wr_len = sizeof *res_wr + sizeof *res; skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) return -ENOMEM; set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = (struct fw_ri_res_wr *)__skb_put(skb, wr_len); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(1) | FW_WR_COMPL(1)); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (unsigned long) &wr_wait; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_RESET; res->u.cq.iqid = cpu_to_be32(cq->cqid); c4iw_init_wr_wait(&wr_wait); ret = c4iw_ofld_send(rdev, skb); if (!ret) { ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); } kfree(cq->sw_queue); dma_free_coherent(&(rdev->lldi.pdev->dev), cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); return ret; }
static void c2_free_cq_buf(struct c2_dev *c2dev, struct c2_mq *mq) { dma_free_coherent(&c2dev->pcidev->dev, mq->q_size * mq->msg_size, mq->msg_pool.host, dma_unmap_addr(mq, mapping)); }
/** * arc_emac_rx - processing of Rx packets. * @ndev: Pointer to the network device. * @budget: How many BDs to process on 1 call. * * returns: Number of processed BDs * * Iterate through Rx BDs and deliver received packages to upper layer. */ static int arc_emac_rx(struct net_device *ndev, int budget) { struct arc_emac_priv *priv = netdev_priv(ndev); unsigned int work_done; for (work_done = 0; work_done < budget; work_done++) { unsigned int *last_rx_bd = &priv->last_rx_bd; struct net_device_stats *stats = &ndev->stats; struct buffer_state *rx_buff = &priv->rx_buff[*last_rx_bd]; struct arc_emac_bd *rxbd = &priv->rxbd[*last_rx_bd]; unsigned int pktlen, info = le32_to_cpu(rxbd->info); struct sk_buff *skb; dma_addr_t addr; if (unlikely((info & OWN_MASK) == FOR_EMAC)) break; /* Make a note that we saw a packet at this BD. * So next time, driver starts from this + 1 */ *last_rx_bd = (*last_rx_bd + 1) % RX_BD_NUM; if (unlikely((info & FIRST_OR_LAST_MASK) != FIRST_OR_LAST_MASK)) { /* We pre-allocate buffers of MTU size so incoming * packets won't be split/chained. */ if (net_ratelimit()) netdev_err(ndev, "incomplete packet received\n"); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); stats->rx_errors++; stats->rx_length_errors++; continue; } /* Prepare the BD for next cycle. netif_receive_skb() * only if new skb was allocated and mapped to avoid holes * in the RX fifo. */ skb = netdev_alloc_skb_ip_align(ndev, EMAC_BUFFER_SIZE); if (unlikely(!skb)) { if (net_ratelimit()) netdev_err(ndev, "cannot allocate skb\n"); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); stats->rx_errors++; stats->rx_dropped++; continue; } addr = dma_map_single(&ndev->dev, (void *)skb->data, EMAC_BUFFER_SIZE, DMA_FROM_DEVICE); if (dma_mapping_error(&ndev->dev, addr)) { if (net_ratelimit()) netdev_err(ndev, "cannot map dma buffer\n"); dev_kfree_skb(skb); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); stats->rx_errors++; stats->rx_dropped++; continue; } /* unmap previosly mapped skb */ dma_unmap_single(&ndev->dev, dma_unmap_addr(rx_buff, addr), dma_unmap_len(rx_buff, len), DMA_FROM_DEVICE); pktlen = info & LEN_MASK; stats->rx_packets++; stats->rx_bytes += pktlen; skb_put(rx_buff->skb, pktlen); rx_buff->skb->dev = ndev; rx_buff->skb->protocol = eth_type_trans(rx_buff->skb, ndev); netif_receive_skb(rx_buff->skb); rx_buff->skb = skb; dma_unmap_addr_set(rx_buff, addr, addr); dma_unmap_len_set(rx_buff, len, EMAC_BUFFER_SIZE); rxbd->data = cpu_to_le32(addr); /* Make sure pointer to data buffer is set */ wmb(); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); } return work_done; }
int __devinit c2_rnic_init(struct c2_dev *c2dev) { int err; u32 qsize, msgsize; void *q1_pages; void *q2_pages; void __iomem *mmio_regs; /* Device capabilities */ c2dev->device_cap_flags = (IB_DEVICE_RESIZE_MAX_WR | IB_DEVICE_CURR_QP_STATE_MOD | IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW); /* Allocate the qptr_array */ c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *)); if (!c2dev->qptr_array) { return -ENOMEM; } /* Inialize the qptr_array */ memset(c2dev->qptr_array, 0, C2_MAX_CQS * sizeof(void *)); c2dev->qptr_array[0] = (void *) &c2dev->req_vq; c2dev->qptr_array[1] = (void *) &c2dev->rep_vq; c2dev->qptr_array[2] = (void *) &c2dev->aeq; /* Initialize data structures */ init_waitqueue_head(&c2dev->req_vq_wo); spin_lock_init(&c2dev->vqlock); spin_lock_init(&c2dev->lock); /* Allocate MQ shared pointer pool for kernel clients. User * mode client pools are hung off the user context */ err = c2_init_mqsp_pool(c2dev, GFP_KERNEL, &c2dev->kern_mqsp_pool); if (err) { goto bail0; } /* Allocate shared pointers for Q0, Q1, and Q2 from * the shared pointer pool. */ c2dev->hint_count = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, &c2dev->hint_count_dma, GFP_KERNEL); c2dev->req_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, &c2dev->req_vq.shared_dma, GFP_KERNEL); c2dev->rep_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, &c2dev->rep_vq.shared_dma, GFP_KERNEL); c2dev->aeq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, &c2dev->aeq.shared_dma, GFP_KERNEL); if (!c2dev->hint_count || !c2dev->req_vq.shared || !c2dev->rep_vq.shared || !c2dev->aeq.shared) { err = -ENOMEM; goto bail1; } mmio_regs = c2dev->kva; /* Initialize the Verbs Request Queue */ c2_mq_req_init(&c2dev->req_vq, 0, be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_QSIZE)), be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_MSGSIZE)), mmio_regs + be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_POOLSTART)), mmio_regs + be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_SHARED)), C2_MQ_ADAPTER_TARGET); /* Initialize the Verbs Reply Queue */ qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_QSIZE)); msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_MSGSIZE)); q1_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize, &c2dev->rep_vq.host_dma, GFP_KERNEL); if (!q1_pages) { err = -ENOMEM; goto bail1; } dma_unmap_addr_set(&c2dev->rep_vq, mapping, c2dev->rep_vq.host_dma); pr_debug("%s rep_vq va %p dma %llx\n", __func__, q1_pages, (unsigned long long) c2dev->rep_vq.host_dma); c2_mq_rep_init(&c2dev->rep_vq, 1, qsize, msgsize, q1_pages, mmio_regs + be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_SHARED)), C2_MQ_HOST_TARGET); /* Initialize the Asynchronus Event Queue */ qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_QSIZE)); msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_MSGSIZE)); q2_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize, &c2dev->aeq.host_dma, GFP_KERNEL); if (!q2_pages) { err = -ENOMEM; goto bail2; } dma_unmap_addr_set(&c2dev->aeq, mapping, c2dev->aeq.host_dma); pr_debug("%s aeq va %p dma %llx\n", __func__, q2_pages, (unsigned long long) c2dev->aeq.host_dma); c2_mq_rep_init(&c2dev->aeq, 2, qsize, msgsize, q2_pages, mmio_regs + be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_SHARED)), C2_MQ_HOST_TARGET); /* Initialize the verbs request allocator */ err = vq_init(c2dev); if (err) goto bail3; /* Enable interrupts on the adapter */ writel(0, c2dev->regs + C2_IDIS); /* create the WR init message */ err = c2_adapter_init(c2dev); if (err) goto bail4; c2dev->init++; /* open an adapter instance */ err = c2_rnic_open(c2dev); if (err) goto bail4; /* Initialize cached the adapter limits */ if (c2_rnic_query(c2dev, &c2dev->props)) goto bail5; /* Initialize the PD pool */ err = c2_init_pd_table(c2dev); if (err) goto bail5; /* Initialize the QP pool */ c2_init_qp_table(c2dev); return 0; bail5: c2_rnic_close(c2dev); bail4: vq_term(c2dev); bail3: dma_free_coherent(&c2dev->pcidev->dev, c2dev->aeq.q_size * c2dev->aeq.msg_size, q2_pages, dma_unmap_addr(&c2dev->aeq, mapping)); bail2: dma_free_coherent(&c2dev->pcidev->dev, c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size, q1_pages, dma_unmap_addr(&c2dev->rep_vq, mapping)); bail1: c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool); bail0: vfree(c2dev->qptr_array); return err; }
static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx, struct c4iw_wr_wait *wr_waitp) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; int user = (uctx != &rdev->uctx); int ret; struct sk_buff *skb; struct c4iw_ucontext *ucontext = NULL; if (user) ucontext = container_of(uctx, struct c4iw_ucontext, uctx); cq->cqid = c4iw_get_cqid(rdev, uctx); if (!cq->cqid) { ret = -ENOMEM; goto err1; } if (!user) { cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL); if (!cq->sw_queue) { ret = -ENOMEM; goto err2; } } cq->queue = dma_alloc_coherent(&rdev->lldi.pdev->dev, cq->memsize, &cq->dma_addr, GFP_KERNEL); if (!cq->queue) { ret = -ENOMEM; goto err3; } dma_unmap_addr_set(cq, mapping, cq->dma_addr); memset(cq->queue, 0, cq->memsize); if (user && ucontext->is_32b_cqe) { cq->qp_errp = &((struct t4_status_page *) ((u8 *)cq->queue + (cq->size - 1) * (sizeof(*cq->queue) / 2)))->qp_err; } else { cq->qp_errp = &((struct t4_status_page *) ((u8 *)cq->queue + (cq->size - 1) * sizeof(*cq->queue)))->qp_err; } /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + sizeof *res; skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto err4; } set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = __skb_put_zero(skb, wr_len); res_wr->op_nres = cpu_to_be32( FW_WR_OP_V(FW_RI_RES_WR) | FW_RI_RES_WR_NRES_V(1) | FW_WR_COMPL_F); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (uintptr_t)wr_waitp; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_WRITE; res->u.cq.iqid = cpu_to_be32(cq->cqid); res->u.cq.iqandst_to_iqandstindex = cpu_to_be32( FW_RI_RES_WR_IQANUS_V(0) | FW_RI_RES_WR_IQANUD_V(1) | FW_RI_RES_WR_IQANDST_F | FW_RI_RES_WR_IQANDSTINDEX_V( rdev->lldi.ciq_ids[cq->vector])); res->u.cq.iqdroprss_to_iqesize = cpu_to_be16( FW_RI_RES_WR_IQDROPRSS_F | FW_RI_RES_WR_IQPCIECH_V(2) | FW_RI_RES_WR_IQINTCNTTHRESH_V(0) | FW_RI_RES_WR_IQO_F | ((user && ucontext->is_32b_cqe) ? FW_RI_RES_WR_IQESIZE_V(1) : FW_RI_RES_WR_IQESIZE_V(2))); res->u.cq.iqsize = cpu_to_be16(cq->size); res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr); c4iw_init_wr_wait(wr_waitp); ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__); if (ret) goto err4; cq->gen = 1; cq->gts = rdev->lldi.gts_reg; cq->rdev = rdev; cq->bar2_va = c4iw_bar2_addrs(rdev, cq->cqid, CXGB4_BAR2_QTYPE_INGRESS, &cq->bar2_qid, user ? &cq->bar2_pa : NULL); if (user && !cq->bar2_pa) { pr_warn("%s: cqid %u not in BAR2 range\n", pci_name(rdev->lldi.pdev), cq->cqid); ret = -EINVAL; goto err4; } return 0; err4: dma_free_coherent(&rdev->lldi.pdev->dev, cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); err3: kfree(cq->sw_queue); err2: c4iw_put_cqid(rdev, cq->cqid, uctx); err1: return ret; }
static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; int user = (uctx != &rdev->uctx); struct c4iw_wr_wait wr_wait; int ret; struct sk_buff *skb; cq->cqid = c4iw_get_cqid(rdev, uctx); if (!cq->cqid) { ret = -ENOMEM; goto err1; } if (!user) { cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL); if (!cq->sw_queue) { ret = -ENOMEM; goto err2; } } cq->queue = dma_alloc_coherent(&rdev->lldi.pdev->dev, cq->memsize, &cq->dma_addr, GFP_KERNEL); if (!cq->queue) { ret = -ENOMEM; goto err3; } dma_unmap_addr_set(cq, mapping, cq->dma_addr); memset(cq->queue, 0, cq->memsize); /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + sizeof *res; skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto err4; } set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = (struct fw_ri_res_wr *)__skb_put(skb, wr_len); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(1) | FW_WR_COMPL(1)); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (unsigned long) &wr_wait; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_WRITE; res->u.cq.iqid = cpu_to_be32(cq->cqid); res->u.cq.iqandst_to_iqandstindex = cpu_to_be32( V_FW_RI_RES_WR_IQANUS(0) | V_FW_RI_RES_WR_IQANUD(1) | F_FW_RI_RES_WR_IQANDST | V_FW_RI_RES_WR_IQANDSTINDEX(*rdev->lldi.rxq_ids)); res->u.cq.iqdroprss_to_iqesize = cpu_to_be16( F_FW_RI_RES_WR_IQDROPRSS | V_FW_RI_RES_WR_IQPCIECH(2) | V_FW_RI_RES_WR_IQINTCNTTHRESH(0) | F_FW_RI_RES_WR_IQO | V_FW_RI_RES_WR_IQESIZE(1)); res->u.cq.iqsize = cpu_to_be16(cq->size); res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr); c4iw_init_wr_wait(&wr_wait); ret = c4iw_ofld_send(rdev, skb); if (ret) goto err4; PDBG("%s wait_event wr_wait %p\n", __func__, &wr_wait); ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); if (ret) goto err4; cq->gen = 1; cq->gts = rdev->lldi.gts_reg; cq->rdev = rdev; if (user) { cq->ugts = (u64)pci_resource_start(rdev->lldi.pdev, 2) + (cq->cqid << rdev->cqshift); cq->ugts &= PAGE_MASK; } return 0; err4: dma_free_coherent(&rdev->lldi.pdev->dev, cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); err3: kfree(cq->sw_queue); err2: c4iw_put_cqid(rdev, cq->cqid, uctx); err1: return ret; }
static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, struct t4_cq *rcq, struct t4_cq *scq, struct c4iw_dev_ucontext *uctx) { int user = (uctx != &rdev->uctx); struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; struct c4iw_wr_wait wr_wait; struct sk_buff *skb; int ret; int eqsize; wq->sq.qid = c4iw_get_qpid(rdev, uctx); if (!wq->sq.qid) return -ENOMEM; wq->rq.qid = c4iw_get_qpid(rdev, uctx); if (!wq->rq.qid) { ret = -ENOMEM; goto free_sq_qid; } if (!user) { wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq, GFP_KERNEL); if (!wq->sq.sw_sq) { ret = -ENOMEM; goto free_rq_qid; } wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq, GFP_KERNEL); if (!wq->rq.sw_rq) { ret = -ENOMEM; goto free_sw_sq; } } /* * RQT must be a power of 2. */ wq->rq.rqt_size = roundup_pow_of_two(wq->rq.size); wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size); if (!wq->rq.rqt_hwaddr) { ret = -ENOMEM; goto free_sw_rq; } if (user) { ret = alloc_oc_sq(rdev, &wq->sq); if (ret) goto free_hwaddr; ret = alloc_host_sq(rdev, &wq->sq); if (ret) goto free_sq; } else ret = alloc_host_sq(rdev, &wq->sq); if (ret) goto free_hwaddr; memset(wq->sq.queue, 0, wq->sq.memsize); dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); wq->rq.queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev), wq->rq.memsize, &(wq->rq.dma_addr), GFP_KERNEL); if (!wq->rq.queue) goto free_sq; PDBG("%s sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx\n", __func__, wq->sq.queue, (unsigned long long)virt_to_phys(wq->sq.queue), wq->rq.queue, (unsigned long long)virt_to_phys(wq->rq.queue)); memset(wq->rq.queue, 0, wq->rq.memsize); dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); wq->db = rdev->lldi.db_reg; wq->gts = rdev->lldi.gts_reg; if (user) { wq->sq.udb = (u64)pci_resource_start(rdev->lldi.pdev, 2) + (wq->sq.qid << rdev->qpshift); wq->sq.udb &= PAGE_MASK; wq->rq.udb = (u64)pci_resource_start(rdev->lldi.pdev, 2) + (wq->rq.qid << rdev->qpshift); wq->rq.udb &= PAGE_MASK; } wq->rdev = rdev; wq->rq.msn = 1; /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + 2 * sizeof *res; skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto free_dma; } set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = (struct fw_ri_res_wr *)__skb_put(skb, wr_len); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(2) | FW_WR_COMPL(1)); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (unsigned long) &wr_wait; res = res_wr->res; res->u.sqrq.restype = FW_RI_RES_TYPE_SQ; res->u.sqrq.op = FW_RI_RES_OP_WRITE; /* * eqsize is the number of 64B entries plus the status page size. */ eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + T4_EQ_STATUS_ENTRIES; res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ (t4_sq_onchip(&wq->sq) ? F_FW_RI_RES_WR_ONCHIP : 0) | V_FW_RI_RES_WR_IQID(scq->cqid)); res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( V_FW_RI_RES_WR_DCAEN(0) | V_FW_RI_RES_WR_DCACPU(0) | V_FW_RI_RES_WR_FBMIN(2) | V_FW_RI_RES_WR_FBMAX(2) | V_FW_RI_RES_WR_CIDXFTHRESHO(0) | V_FW_RI_RES_WR_CIDXFTHRESH(0) | V_FW_RI_RES_WR_EQSIZE(eqsize)); res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid); res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr); res++; res->u.sqrq.restype = FW_RI_RES_TYPE_RQ; res->u.sqrq.op = FW_RI_RES_OP_WRITE; /* * eqsize is the number of 64B entries plus the status page size. */ eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + T4_EQ_STATUS_ENTRIES; res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ V_FW_RI_RES_WR_IQID(rcq->cqid)); res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( V_FW_RI_RES_WR_DCAEN(0) | V_FW_RI_RES_WR_DCACPU(0) | V_FW_RI_RES_WR_FBMIN(2) | V_FW_RI_RES_WR_FBMAX(2) | V_FW_RI_RES_WR_CIDXFTHRESHO(0) | V_FW_RI_RES_WR_CIDXFTHRESH(0) | V_FW_RI_RES_WR_EQSIZE(eqsize)); res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid); res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr); c4iw_init_wr_wait(&wr_wait); ret = c4iw_ofld_send(rdev, skb); if (ret) goto free_dma; ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__); if (ret) goto free_dma; PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx\n", __func__, wq->sq.qid, wq->rq.qid, wq->db, (unsigned long long)wq->sq.udb, (unsigned long long)wq->rq.udb); return 0; free_dma: dma_free_coherent(&(rdev->lldi.pdev->dev), wq->rq.memsize, wq->rq.queue, dma_unmap_addr(&wq->rq, mapping)); free_sq: dealloc_sq(rdev, &wq->sq); free_hwaddr: c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); free_sw_rq: kfree(wq->rq.sw_rq); free_sw_sq: kfree(wq->sq.sw_sq); free_rq_qid: c4iw_put_qpid(rdev, wq->rq.qid, uctx); free_sq_qid: c4iw_put_qpid(rdev, wq->sq.qid, uctx); return ret; }