static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *tx_bi) { xdp_return_frame(tx_bi->xdpf); dma_unmap_single(tx_ring->dev, dma_unmap_addr(tx_bi, dma), dma_unmap_len(tx_bi, len), DMA_TO_DEVICE); dma_unmap_len_set(tx_bi, len, 0); }
/** * arc_emac_tx - Starts the data transmission. * @skb: sk_buff pointer that contains data to be Transmitted. * @ndev: Pointer to net_device structure. * * returns: NETDEV_TX_OK, on success * NETDEV_TX_BUSY, if any of the descriptors are not free. * * This function is invoked from upper layers to initiate transmission. */ static int arc_emac_tx(struct sk_buff *skb, struct net_device *ndev) { struct arc_emac_priv *priv = netdev_priv(ndev); unsigned int len, *txbd_curr = &priv->txbd_curr; struct net_device_stats *stats = &priv->stats; __le32 *info = &priv->txbd[*txbd_curr].info; dma_addr_t addr; if (skb_padto(skb, ETH_ZLEN)) return NETDEV_TX_OK; len = max_t(unsigned int, ETH_ZLEN, skb->len); /* EMAC still holds this buffer in its possession. * CPU must not modify this buffer descriptor */ if (unlikely((le32_to_cpu(*info) & OWN_MASK) == FOR_EMAC)) { netif_stop_queue(ndev); return NETDEV_TX_BUSY; } addr = dma_map_single(&ndev->dev, (void *)skb->data, len, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(&ndev->dev, addr))) { stats->tx_dropped++; stats->tx_errors++; dev_kfree_skb(skb); return NETDEV_TX_OK; } dma_unmap_addr_set(&priv->tx_buff[*txbd_curr], addr, addr); dma_unmap_len_set(&priv->tx_buff[*txbd_curr], len, len); priv->tx_buff[*txbd_curr].skb = skb; priv->txbd[*txbd_curr].data = cpu_to_le32(addr); /* Make sure pointer to data buffer is set */ wmb(); skb_tx_timestamp(skb); *info = cpu_to_le32(FOR_EMAC | FIRST_OR_LAST_MASK | len); /* Increment index to point to the next BD */ *txbd_curr = (*txbd_curr + 1) % TX_BD_NUM; /* Get "info" of the next BD */ info = &priv->txbd[*txbd_curr].info; /* Check if if Tx BD ring is full - next BD is still owned by EMAC */ if (unlikely((le32_to_cpu(*info) & OWN_MASK) == FOR_EMAC)) netif_stop_queue(ndev); arc_reg_set(priv, R_STATUS, TXPL_MASK); return NETDEV_TX_OK; }
void fm10k_unmap_and_free_tx_resource(struct fm10k_ring *ring, struct fm10k_tx_buffer *tx_buffer) { if (tx_buffer->skb) { dev_kfree_skb_any(tx_buffer->skb); if (dma_unmap_len(tx_buffer, len)) dma_unmap_single(ring->dev, dma_unmap_addr(tx_buffer, dma), dma_unmap_len(tx_buffer, len), DMA_TO_DEVICE); } else if (dma_unmap_len(tx_buffer, len)) { dma_unmap_page(ring->dev, dma_unmap_addr(tx_buffer, dma), dma_unmap_len(tx_buffer, len), DMA_TO_DEVICE); } tx_buffer->next_to_watch = NULL; tx_buffer->skb = NULL; dma_unmap_len_set(tx_buffer, len, 0); /* tx_buffer must be completely set up in the transmit path */ }
/** * arc_emac_open - Open the network device. * @ndev: Pointer to the network device. * * returns: 0, on success or non-zero error value on failure. * * This function sets the MAC address, requests and enables an IRQ * for the EMAC device and starts the Tx queue. * It also connects to the phy device. */ static int arc_emac_open(struct net_device *ndev) { struct arc_emac_priv *priv = netdev_priv(ndev); struct phy_device *phy_dev = ndev->phydev; int i; phy_dev->autoneg = AUTONEG_ENABLE; phy_dev->speed = 0; phy_dev->duplex = 0; linkmode_and(phy_dev->advertising, phy_dev->advertising, phy_dev->supported); priv->last_rx_bd = 0; /* Allocate and set buffers for Rx BD's */ for (i = 0; i < RX_BD_NUM; i++) { dma_addr_t addr; unsigned int *last_rx_bd = &priv->last_rx_bd; struct arc_emac_bd *rxbd = &priv->rxbd[*last_rx_bd]; struct buffer_state *rx_buff = &priv->rx_buff[*last_rx_bd]; rx_buff->skb = netdev_alloc_skb_ip_align(ndev, EMAC_BUFFER_SIZE); if (unlikely(!rx_buff->skb)) return -ENOMEM; addr = dma_map_single(&ndev->dev, (void *)rx_buff->skb->data, EMAC_BUFFER_SIZE, DMA_FROM_DEVICE); if (dma_mapping_error(&ndev->dev, addr)) { netdev_err(ndev, "cannot dma map\n"); dev_kfree_skb(rx_buff->skb); return -ENOMEM; } dma_unmap_addr_set(rx_buff, addr, addr); dma_unmap_len_set(rx_buff, len, EMAC_BUFFER_SIZE); rxbd->data = cpu_to_le32(addr); /* Make sure pointer to data buffer is set */ wmb(); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); *last_rx_bd = (*last_rx_bd + 1) % RX_BD_NUM; } priv->txbd_curr = 0; priv->txbd_dirty = 0; /* Clean Tx BD's */ memset(priv->txbd, 0, TX_RING_SZ); /* Initialize logical address filter */ arc_reg_set(priv, R_LAFL, 0); arc_reg_set(priv, R_LAFH, 0); /* Set BD ring pointers for device side */ arc_reg_set(priv, R_RX_RING, (unsigned int)priv->rxbd_dma); arc_reg_set(priv, R_TX_RING, (unsigned int)priv->txbd_dma); /* Enable interrupts */ arc_reg_set(priv, R_ENABLE, RXINT_MASK | TXINT_MASK | ERR_MASK); /* Set CONTROL */ arc_reg_set(priv, R_CTRL, (RX_BD_NUM << 24) | /* RX BD table length */ (TX_BD_NUM << 16) | /* TX BD table length */ TXRN_MASK | RXRN_MASK); napi_enable(&priv->napi); /* Enable EMAC */ arc_reg_or(priv, R_CTRL, EN_MASK); phy_start(ndev->phydev); netif_start_queue(ndev); return 0; }
/** * arc_emac_rx - processing of Rx packets. * @ndev: Pointer to the network device. * @budget: How many BDs to process on 1 call. * * returns: Number of processed BDs * * Iterate through Rx BDs and deliver received packages to upper layer. */ static int arc_emac_rx(struct net_device *ndev, int budget) { struct arc_emac_priv *priv = netdev_priv(ndev); unsigned int work_done; for (work_done = 0; work_done < budget; work_done++) { unsigned int *last_rx_bd = &priv->last_rx_bd; struct net_device_stats *stats = &ndev->stats; struct buffer_state *rx_buff = &priv->rx_buff[*last_rx_bd]; struct arc_emac_bd *rxbd = &priv->rxbd[*last_rx_bd]; unsigned int pktlen, info = le32_to_cpu(rxbd->info); struct sk_buff *skb; dma_addr_t addr; if (unlikely((info & OWN_MASK) == FOR_EMAC)) break; /* Make a note that we saw a packet at this BD. * So next time, driver starts from this + 1 */ *last_rx_bd = (*last_rx_bd + 1) % RX_BD_NUM; if (unlikely((info & FIRST_OR_LAST_MASK) != FIRST_OR_LAST_MASK)) { /* We pre-allocate buffers of MTU size so incoming * packets won't be split/chained. */ if (net_ratelimit()) netdev_err(ndev, "incomplete packet received\n"); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); stats->rx_errors++; stats->rx_length_errors++; continue; } /* Prepare the BD for next cycle. netif_receive_skb() * only if new skb was allocated and mapped to avoid holes * in the RX fifo. */ skb = netdev_alloc_skb_ip_align(ndev, EMAC_BUFFER_SIZE); if (unlikely(!skb)) { if (net_ratelimit()) netdev_err(ndev, "cannot allocate skb\n"); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); stats->rx_errors++; stats->rx_dropped++; continue; } addr = dma_map_single(&ndev->dev, (void *)skb->data, EMAC_BUFFER_SIZE, DMA_FROM_DEVICE); if (dma_mapping_error(&ndev->dev, addr)) { if (net_ratelimit()) netdev_err(ndev, "cannot map dma buffer\n"); dev_kfree_skb(skb); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); stats->rx_errors++; stats->rx_dropped++; continue; } /* unmap previosly mapped skb */ dma_unmap_single(&ndev->dev, dma_unmap_addr(rx_buff, addr), dma_unmap_len(rx_buff, len), DMA_FROM_DEVICE); pktlen = info & LEN_MASK; stats->rx_packets++; stats->rx_bytes += pktlen; skb_put(rx_buff->skb, pktlen); rx_buff->skb->dev = ndev; rx_buff->skb->protocol = eth_type_trans(rx_buff->skb, ndev); netif_receive_skb(rx_buff->skb); rx_buff->skb = skb; dma_unmap_addr_set(rx_buff, addr, addr); dma_unmap_len_set(rx_buff, len, EMAC_BUFFER_SIZE); rxbd->data = cpu_to_le32(addr); /* Make sure pointer to data buffer is set */ wmb(); /* Return ownership to EMAC */ rxbd->info = cpu_to_le32(FOR_EMAC | EMAC_BUFFER_SIZE); } return work_done; }
/** * iwl_enqueue_hcmd - enqueue a uCode command * @priv: device private data point * @cmd: a point to the ucode command structure * * The function returns < 0 values to indicate the operation is * failed. On success, it turns the index (> 0) of command in the * command queue. */ static int iwl_enqueue_hcmd(struct iwl_priv *priv, struct iwl_host_cmd *cmd) { struct iwl_tx_queue *txq = &priv->txq[priv->cmd_queue]; struct iwl_queue *q = &txq->q; struct iwl_device_cmd *out_cmd; struct iwl_cmd_meta *out_meta; dma_addr_t phys_addr; unsigned long flags; u32 idx; u16 copy_size, cmd_size; bool is_ct_kill = false; bool had_nocopy = false; int i; u8 *cmd_dest; #ifdef CONFIG_IWLWIFI_DEVICE_TRACING const void *trace_bufs[IWL_MAX_CMD_TFDS + 1] = {}; int trace_lens[IWL_MAX_CMD_TFDS + 1] = {}; int trace_idx; #endif if (test_bit(STATUS_FW_ERROR, &priv->status)) { IWL_WARN(priv, "fw recovery, no hcmd send\n"); return -EIO; } if ((priv->ucode_owner == IWL_OWNERSHIP_TM) && !(cmd->flags & CMD_ON_DEMAND)) { IWL_DEBUG_HC(priv, "tm own the uCode, no regular hcmd send\n"); return -EIO; } copy_size = sizeof(out_cmd->hdr); cmd_size = sizeof(out_cmd->hdr); /* need one for the header if the first is NOCOPY */ BUILD_BUG_ON(IWL_MAX_CMD_TFDS > IWL_NUM_OF_TBS - 1); for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { had_nocopy = true; } else { /* NOCOPY must not be followed by normal! */ if (WARN_ON(had_nocopy)) return -EINVAL; copy_size += cmd->len[i]; } cmd_size += cmd->len[i]; } /* * If any of the command structures end up being larger than * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically * allocated into separate TFDs, then we will need to * increase the size of the buffers. */ if (WARN_ON(copy_size > TFD_MAX_PAYLOAD_SIZE)) return -EINVAL; if (iwl_is_rfkill(priv) || iwl_is_ctkill(priv)) { IWL_WARN(priv, "Not sending command - %s KILL\n", iwl_is_rfkill(priv) ? "RF" : "CT"); return -EIO; } spin_lock_irqsave(&priv->hcmd_lock, flags); if (iwl_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { spin_unlock_irqrestore(&priv->hcmd_lock, flags); IWL_ERR(priv, "No space in command queue\n"); is_ct_kill = iwl_check_for_ct_kill(priv); if (!is_ct_kill) { IWL_ERR(priv, "Restarting adapter due to queue full\n"); iwlagn_fw_error(priv, false); } return -ENOSPC; } idx = get_cmd_index(q, q->write_ptr); out_cmd = txq->cmd[idx]; out_meta = &txq->meta[idx]; memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ if (cmd->flags & CMD_WANT_SKB) out_meta->source = cmd; if (cmd->flags & CMD_ASYNC) out_meta->callback = cmd->callback; /* set up the header */ out_cmd->hdr.cmd = cmd->id; out_cmd->hdr.flags = 0; out_cmd->hdr.sequence = cpu_to_le16(QUEUE_TO_SEQ(priv->cmd_queue) | INDEX_TO_SEQ(q->write_ptr)); /* and copy the data that needs to be copied */ cmd_dest = &out_cmd->cmd.payload[0]; for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) break; memcpy(cmd_dest, cmd->data[i], cmd->len[i]); cmd_dest += cmd->len[i]; } IWL_DEBUG_HC(priv, "Sending command %s (#%x), seq: 0x%04X, " "%d bytes at %d[%d]:%d\n", get_cmd_string(out_cmd->hdr.cmd), out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), cmd_size, q->write_ptr, idx, priv->cmd_queue); phys_addr = dma_map_single(priv->bus->dev, &out_cmd->hdr, copy_size, DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(priv->bus->dev, phys_addr))) { idx = -ENOMEM; goto out; } dma_unmap_addr_set(out_meta, mapping, phys_addr); dma_unmap_len_set(out_meta, len, copy_size); iwlagn_txq_attach_buf_to_tfd(priv, txq, phys_addr, copy_size, 1); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[0] = &out_cmd->hdr; trace_lens[0] = copy_size; trace_idx = 1; #endif for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (!(cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY)) continue; phys_addr = dma_map_single(priv->bus->dev, (void *)cmd->data[i], cmd->len[i], DMA_BIDIRECTIONAL); if (dma_mapping_error(priv->bus->dev, phys_addr)) { iwlagn_unmap_tfd(priv, out_meta, &txq->tfds[q->write_ptr], DMA_BIDIRECTIONAL); idx = -ENOMEM; goto out; } iwlagn_txq_attach_buf_to_tfd(priv, txq, phys_addr, cmd->len[i], 0); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[trace_idx] = cmd->data[i]; trace_lens[trace_idx] = cmd->len[i]; trace_idx++; #endif } out_meta->flags = cmd->flags; txq->need_update = 1; /* check that tracing gets all possible blocks */ BUILD_BUG_ON(IWL_MAX_CMD_TFDS + 1 != 3); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_iwlwifi_dev_hcmd(priv, cmd->flags, trace_bufs[0], trace_lens[0], trace_bufs[1], trace_lens[1], trace_bufs[2], trace_lens[2]); #endif /* Increment and update queue's write index */ q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd); iwl_txq_update_write_ptr(priv, txq); out: spin_unlock_irqrestore(&priv->hcmd_lock, flags); return idx; }
/** * iwl_enqueue_hcmd - enqueue a uCode command * @priv: device private data point * @cmd: a point to the ucode command structure * * The function returns < 0 values to indicate the operation is * failed. On success, it turns the index (> 0) of command in the * command queue. */ static int iwl_enqueue_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_tx_queue *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; struct iwl_queue *q = &txq->q; struct iwl_device_cmd *out_cmd; struct iwl_cmd_meta *out_meta; dma_addr_t phys_addr; u32 idx; u16 copy_size, cmd_size; bool had_nocopy = false; int i; u8 *cmd_dest; #ifdef CONFIG_IWLWIFI_DEVICE_TRACING const void *trace_bufs[IWL_MAX_CMD_TFDS + 1] = {}; int trace_lens[IWL_MAX_CMD_TFDS + 1] = {}; int trace_idx; #endif copy_size = sizeof(out_cmd->hdr); cmd_size = sizeof(out_cmd->hdr); /* need one for the header if the first is NOCOPY */ BUILD_BUG_ON(IWL_MAX_CMD_TFDS > IWL_NUM_OF_TBS - 1); for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { had_nocopy = true; } else { /* NOCOPY must not be followed by normal! */ if (WARN_ON(had_nocopy)) return -EINVAL; copy_size += cmd->len[i]; } cmd_size += cmd->len[i]; } /* * If any of the command structures end up being larger than * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically * allocated into separate TFDs, then we will need to * increase the size of the buffers. */ if (WARN_ON(copy_size > TFD_MAX_PAYLOAD_SIZE)) return -EINVAL; spin_lock_bh(&txq->lock); if (iwl_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { spin_unlock_bh(&txq->lock); IWL_ERR(trans, "No space in command queue\n"); iwl_op_mode_cmd_queue_full(trans->op_mode); return -ENOSPC; } idx = get_cmd_index(q, q->write_ptr); out_cmd = txq->entries[idx].cmd; out_meta = &txq->entries[idx].meta; memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ if (cmd->flags & CMD_WANT_SKB) out_meta->source = cmd; /* set up the header */ out_cmd->hdr.cmd = cmd->id; out_cmd->hdr.flags = 0; out_cmd->hdr.sequence = cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | INDEX_TO_SEQ(q->write_ptr)); /* and copy the data that needs to be copied */ cmd_dest = out_cmd->payload; for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) break; memcpy(cmd_dest, cmd->data[i], cmd->len[i]); cmd_dest += cmd->len[i]; } IWL_DEBUG_HC(trans, "Sending command %s (#%x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", trans_pcie_get_cmd_string(trans_pcie, out_cmd->hdr.cmd), out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), cmd_size, q->write_ptr, idx, trans_pcie->cmd_queue); phys_addr = dma_map_single(trans->dev, &out_cmd->hdr, copy_size, DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(trans->dev, phys_addr))) { idx = -ENOMEM; goto out; } dma_unmap_addr_set(out_meta, mapping, phys_addr); dma_unmap_len_set(out_meta, len, copy_size); iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr, copy_size, 1); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[0] = &out_cmd->hdr; trace_lens[0] = copy_size; trace_idx = 1; #endif for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (!(cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY)) continue; phys_addr = dma_map_single(trans->dev, (void *)cmd->data[i], cmd->len[i], DMA_BIDIRECTIONAL); if (dma_mapping_error(trans->dev, phys_addr)) { iwl_unmap_tfd(trans, out_meta, &txq->tfds[q->write_ptr], DMA_BIDIRECTIONAL); idx = -ENOMEM; goto out; } iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr, cmd->len[i], 0); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[trace_idx] = cmd->data[i]; trace_lens[trace_idx] = cmd->len[i]; trace_idx++; #endif } out_meta->flags = cmd->flags; txq->need_update = 1; /* check that tracing gets all possible blocks */ BUILD_BUG_ON(IWL_MAX_CMD_TFDS + 1 != 3); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_iwlwifi_dev_hcmd(trans->dev, cmd->flags, trace_bufs[0], trace_lens[0], trace_bufs[1], trace_lens[1], trace_bufs[2], trace_lens[2]); #endif /* start timer if queue currently empty */ if (q->read_ptr == q->write_ptr && trans_pcie->wd_timeout) mod_timer(&txq->stuck_timer, jiffies + trans_pcie->wd_timeout); /* Increment and update queue's write index */ q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd); iwl_txq_update_write_ptr(trans, txq); out: spin_unlock_bh(&txq->lock); return idx; }
/** * iwl_legacy_enqueue_hcmd - enqueue a uCode command * @priv: device private data point * @cmd: a point to the ucode command structure * * The function returns < 0 values to indicate the operation is * failed. On success, it turns the index (> 0) of command in the * command queue. */ int iwl_legacy_enqueue_hcmd(struct iwl_priv *priv, struct iwl_host_cmd *cmd) { struct iwl_tx_queue *txq = &priv->txq[priv->cmd_queue]; struct iwl_queue *q = &txq->q; struct iwl_device_cmd *out_cmd; struct iwl_cmd_meta *out_meta; dma_addr_t phys_addr; unsigned long flags; int len; u32 idx; u16 fix_size; cmd->len = priv->cfg->ops->utils->get_hcmd_size(cmd->id, cmd->len); fix_size = (u16)(cmd->len + sizeof(out_cmd->hdr)); /* If any of the command structures end up being larger than * the TFD_MAX_PAYLOAD_SIZE, and it sent as a 'small' command then * we will need to increase the size of the TFD entries * Also, check to see if command buffer should not exceed the size * of device_cmd and max_cmd_size. */ BUG_ON((fix_size > TFD_MAX_PAYLOAD_SIZE) && !(cmd->flags & CMD_SIZE_HUGE)); BUG_ON(fix_size > IWL_MAX_CMD_SIZE); if (iwl_legacy_is_rfkill(priv) || iwl_legacy_is_ctkill(priv)) { IWL_WARN(priv, "Not sending command - %s KILL\n", iwl_legacy_is_rfkill(priv) ? "RF" : "CT"); return -EIO; } spin_lock_irqsave(&priv->hcmd_lock, flags); if (iwl_legacy_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { spin_unlock_irqrestore(&priv->hcmd_lock, flags); IWL_ERR(priv, "Restarting adapter due to command queue full\n"); queue_work(priv->workqueue, &priv->restart); return -ENOSPC; } idx = iwl_legacy_get_cmd_index(q, q->write_ptr, cmd->flags & CMD_SIZE_HUGE); out_cmd = txq->cmd[idx]; out_meta = &txq->meta[idx]; if (WARN_ON(out_meta->flags & CMD_MAPPED)) { spin_unlock_irqrestore(&priv->hcmd_lock, flags); return -ENOSPC; } memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ out_meta->flags = cmd->flags | CMD_MAPPED; if (cmd->flags & CMD_WANT_SKB) out_meta->source = cmd; if (cmd->flags & CMD_ASYNC) out_meta->callback = cmd->callback; out_cmd->hdr.cmd = cmd->id; memcpy(&out_cmd->cmd.payload, cmd->data, cmd->len); /* At this point, the out_cmd now has all of the incoming cmd * information */ out_cmd->hdr.flags = 0; out_cmd->hdr.sequence = cpu_to_le16(QUEUE_TO_SEQ(priv->cmd_queue) | INDEX_TO_SEQ(q->write_ptr)); if (cmd->flags & CMD_SIZE_HUGE) out_cmd->hdr.sequence |= SEQ_HUGE_FRAME; len = sizeof(struct iwl_device_cmd); if (idx == TFD_CMD_SLOTS) len = IWL_MAX_CMD_SIZE; #ifdef CONFIG_IWLWIFI_LEGACY_DEBUG switch (out_cmd->hdr.cmd) { case REPLY_TX_LINK_QUALITY_CMD: case SENSITIVITY_CMD: IWL_DEBUG_HC_DUMP(priv, "Sending command %s (#%x), seq: 0x%04X, " "%d bytes at %d[%d]:%d\n", iwl_legacy_get_cmd_string(out_cmd->hdr.cmd), out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), fix_size, q->write_ptr, idx, priv->cmd_queue); break; default: IWL_DEBUG_HC(priv, "Sending command %s (#%x), seq: 0x%04X, " "%d bytes at %d[%d]:%d\n", iwl_legacy_get_cmd_string(out_cmd->hdr.cmd), out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), fix_size, q->write_ptr, idx, priv->cmd_queue); } #endif txq->need_update = 1; if (priv->cfg->ops->lib->txq_update_byte_cnt_tbl) /* Set up entry in queue's byte count circular buffer */ priv->cfg->ops->lib->txq_update_byte_cnt_tbl(priv, txq, 0); phys_addr = pci_map_single(priv->pci_dev, &out_cmd->hdr, fix_size, PCI_DMA_BIDIRECTIONAL); dma_unmap_addr_set(out_meta, mapping, phys_addr); dma_unmap_len_set(out_meta, len, fix_size); trace_iwlwifi_legacy_dev_hcmd(priv, &out_cmd->hdr, fix_size, cmd->flags); priv->cfg->ops->lib->txq_attach_buf_to_tfd(priv, txq, phys_addr, fix_size, 1, U32_PAD(cmd->len)); /* Increment and update queue's write index */ q->write_ptr = iwl_legacy_queue_inc_wrap(q->write_ptr, q->n_bd); iwl_legacy_txq_update_write_ptr(priv, txq); spin_unlock_irqrestore(&priv->hcmd_lock, flags); return idx; }
/** * iwl_enqueue_hcmd - enqueue a uCode command * @priv: device private data point * @cmd: a point to the ucode command structure * * The function returns < 0 values to indicate the operation is * failed. On success, it turns the index (> 0) of command in the * command queue. */ static int iwl_enqueue_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_tx_queue *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; struct iwl_queue *q = &txq->q; struct iwl_device_cmd *out_cmd; struct iwl_cmd_meta *out_meta; dma_addr_t phys_addr; u32 idx; u16 copy_size, cmd_size, dma_size; bool had_nocopy = false; int i; u8 *cmd_dest; const u8 *cmddata[IWL_MAX_CMD_TFDS]; u16 cmdlen[IWL_MAX_CMD_TFDS]; #ifdef CONFIG_IWLWIFI_DEVICE_TRACING const void *trace_bufs[IWL_MAX_CMD_TFDS + 1] = {}; int trace_lens[IWL_MAX_CMD_TFDS + 1] = {}; int trace_idx; #endif if (test_bit(STATUS_FW_ERROR, &trans->shrd->status)) { IWL_WARN(trans, "fw recovery, no hcmd send\n"); return -EIO; } copy_size = sizeof(out_cmd->hdr); cmd_size = sizeof(out_cmd->hdr); /* need one for the header if the first is NOCOPY */ BUILD_BUG_ON(IWL_MAX_CMD_TFDS > IWL_NUM_OF_TBS - 1); for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { cmddata[i] = cmd->data[i]; cmdlen[i] = cmd->len[i]; if (!cmd->len[i]) continue; /* need at least IWL_HCMD_MIN_COPY_SIZE copied */ if (copy_size < IWL_HCMD_MIN_COPY_SIZE) { int copy = IWL_HCMD_MIN_COPY_SIZE - copy_size; if (copy > cmdlen[i]) copy = cmdlen[i]; cmdlen[i] -= copy; cmddata[i] += copy; copy_size += copy; } if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { had_nocopy = true; } else { /* NOCOPY must not be followed by normal! */ if (WARN_ON(had_nocopy)) return -EINVAL; copy_size += cmdlen[i]; } cmd_size += cmd->len[i]; } /* * If any of the command structures end up being larger than * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically * allocated into separate TFDs, then we will need to * increase the size of the buffers. */ if (WARN_ON(copy_size > TFD_MAX_PAYLOAD_SIZE)) return -EINVAL; spin_lock_bh(&txq->lock); if (iwl_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { spin_unlock_bh(&txq->lock); IWL_ERR(trans, "No space in command queue\n"); iwl_op_mode_cmd_queue_full(trans->op_mode); return -ENOSPC; } idx = get_cmd_index(q, q->write_ptr); out_cmd = txq->cmd[idx]; out_meta = &txq->meta[idx]; memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ if (cmd->flags & CMD_WANT_SKB) out_meta->source = cmd; /* set up the header */ out_cmd->hdr.cmd = cmd->id; out_cmd->hdr.flags = 0; out_cmd->hdr.sequence = cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | INDEX_TO_SEQ(q->write_ptr)); /* and copy the data that needs to be copied */ cmd_dest = out_cmd->payload; copy_size = sizeof(out_cmd->hdr); for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { int copy = 0; if (!cmd->len) continue; /* need at least IWL_HCMD_MIN_COPY_SIZE copied */ if (copy_size < IWL_HCMD_MIN_COPY_SIZE) { copy = IWL_HCMD_MIN_COPY_SIZE - copy_size; if (copy > cmd->len[i]) copy = cmd->len[i]; } /* copy everything if not nocopy/dup */ if (!(cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY)) copy = cmd->len[i]; if (copy) { memcpy(cmd_dest, cmd->data[i], copy); cmd_dest += copy; copy_size += copy; } } IWL_DEBUG_HC(trans, "Sending command %s (#%x), seq: 0x%04X, " "%d bytes at %d[%d]:%d\n", get_cmd_string(out_cmd->hdr.cmd), out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), cmd_size, q->write_ptr, idx, trans_pcie->cmd_queue); /* * If the entire command is smaller than IWL_HCMD_MIN_COPY_SIZE, we must * still map at least that many bytes for the hardware to write back to. * We have enough space, so that's not a problem. */ dma_size = max_t(u16, copy_size, IWL_HCMD_MIN_COPY_SIZE); phys_addr = dma_map_single(trans->dev, &out_cmd->hdr, dma_size, DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(trans->dev, phys_addr))) { idx = -ENOMEM; goto out; } dma_unmap_addr_set(out_meta, mapping, phys_addr); dma_unmap_len_set(out_meta, len, dma_size); iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr, copy_size, 1); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[0] = &out_cmd->hdr; trace_lens[0] = copy_size; trace_idx = 1; #endif /* map the remaining (adjusted) nocopy/dup fragments */ for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmdlen[i]) continue; if (!(cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY)) continue; phys_addr = dma_map_single(trans->dev, (void *)cmddata[i], cmdlen[i], DMA_BIDIRECTIONAL); if (dma_mapping_error(trans->dev, phys_addr)) { iwlagn_unmap_tfd(trans, out_meta, &txq->tfds[q->write_ptr], DMA_BIDIRECTIONAL); idx = -ENOMEM; goto out; } iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr, cmdlen[i], 0); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[trace_idx] = cmddata[i]; trace_lens[trace_idx] = cmdlen[i]; trace_idx++; #endif } out_meta->flags = cmd->flags; txq->need_update = 1; /* check that tracing gets all possible blocks */ BUILD_BUG_ON(IWL_MAX_CMD_TFDS + 1 != 3); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_iwlwifi_dev_hcmd(trans->dev, cmd->flags, trace_bufs[0], trace_lens[0], trace_bufs[1], trace_lens[1], trace_bufs[2], trace_lens[2]); #endif /* Increment and update queue's write index */ q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd); iwl_txq_update_write_ptr(trans, txq); out: spin_unlock_bh(&txq->lock); return idx; }
static int iwl_enqueue_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_tx_queue *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; struct iwl_queue *q = &txq->q; struct iwl_device_cmd *out_cmd; struct iwl_cmd_meta *out_meta; dma_addr_t phys_addr; u32 idx; u16 copy_size, cmd_size; bool had_nocopy = false; int i; u8 *cmd_dest; #ifdef CONFIG_IWLWIFI_DEVICE_TRACING const void *trace_bufs[IWL_MAX_CMD_TFDS + 1] = {}; int trace_lens[IWL_MAX_CMD_TFDS + 1] = {}; int trace_idx; #endif if (test_bit(STATUS_FW_ERROR, &trans->shrd->status)) { IWL_WARN(trans, "fw recovery, no hcmd send\n"); return -EIO; } copy_size = sizeof(out_cmd->hdr); cmd_size = sizeof(out_cmd->hdr); BUILD_BUG_ON(IWL_MAX_CMD_TFDS > IWL_NUM_OF_TBS - 1); for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { had_nocopy = true; } else { if (WARN_ON(had_nocopy)) return -EINVAL; copy_size += cmd->len[i]; } cmd_size += cmd->len[i]; } if (WARN_ON(copy_size > TFD_MAX_PAYLOAD_SIZE)) return -EINVAL; spin_lock_bh(&txq->lock); if (iwl_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { spin_unlock_bh(&txq->lock); IWL_ERR(trans, "No space in command queue\n"); iwl_op_mode_cmd_queue_full(trans->op_mode); return -ENOSPC; } idx = get_cmd_index(q, q->write_ptr); out_cmd = txq->cmd[idx]; out_meta = &txq->meta[idx]; memset(out_meta, 0, sizeof(*out_meta)); if (cmd->flags & CMD_WANT_SKB) out_meta->source = cmd; out_cmd->hdr.cmd = cmd->id; out_cmd->hdr.flags = 0; out_cmd->hdr.sequence = cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | INDEX_TO_SEQ(q->write_ptr)); cmd_dest = out_cmd->payload; for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) break; memcpy(cmd_dest, cmd->data[i], cmd->len[i]); cmd_dest += cmd->len[i]; } IWL_DEBUG_HC(trans, "Sending command %s (#%x), seq: 0x%04X, " "%d bytes at %d[%d]:%d\n", get_cmd_string(out_cmd->hdr.cmd), out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), cmd_size, q->write_ptr, idx, trans_pcie->cmd_queue); phys_addr = dma_map_single(trans->dev, &out_cmd->hdr, copy_size, DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(trans->dev, phys_addr))) { idx = -ENOMEM; goto out; } dma_unmap_addr_set(out_meta, mapping, phys_addr); dma_unmap_len_set(out_meta, len, copy_size); iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr, copy_size, 1); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[0] = &out_cmd->hdr; trace_lens[0] = copy_size; trace_idx = 1; #endif for (i = 0; i < IWL_MAX_CMD_TFDS; i++) { if (!cmd->len[i]) continue; if (!(cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY)) continue; phys_addr = dma_map_single(trans->dev, (void *)cmd->data[i], cmd->len[i], DMA_BIDIRECTIONAL); if (dma_mapping_error(trans->dev, phys_addr)) { iwlagn_unmap_tfd(trans, out_meta, &txq->tfds[q->write_ptr], DMA_BIDIRECTIONAL); idx = -ENOMEM; goto out; } iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr, cmd->len[i], 0); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_bufs[trace_idx] = cmd->data[i]; trace_lens[trace_idx] = cmd->len[i]; trace_idx++; #endif } out_meta->flags = cmd->flags; txq->need_update = 1; BUILD_BUG_ON(IWL_MAX_CMD_TFDS + 1 != 3); #ifdef CONFIG_IWLWIFI_DEVICE_TRACING trace_iwlwifi_dev_hcmd(trans->dev, cmd->flags, trace_bufs[0], trace_lens[0], trace_bufs[1], trace_lens[1], trace_bufs[2], trace_lens[2]); #endif q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd); iwl_txq_update_write_ptr(trans, txq); out: spin_unlock_bh(&txq->lock); return idx; }