static void iwl_pcie_rx_replenish_work(struct work_struct *data) { struct iwl_trans_pcie *trans_pcie = container_of(data, struct iwl_trans_pcie, rx_replenish); iwl_pcie_rx_replenish(trans_pcie->trans); }
int iwl_pcie_rx_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; struct iwl_rb_allocator *rba = &trans_pcie->rba; int i, err; if (!rxq->bd) { err = iwl_pcie_rx_alloc(trans); if (err) return err; } if (!rba->alloc_wq) rba->alloc_wq = alloc_workqueue("rb_allocator", WQ_HIGHPRI | WQ_UNBOUND, 1); INIT_WORK(&rba->rx_alloc, iwl_pcie_rx_allocator_work); spin_lock(&rba->lock); atomic_set(&rba->req_pending, 0); atomic_set(&rba->req_ready, 0); /* free all first - we might be reconfigured for a different size */ iwl_pcie_rx_free_rba(trans); iwl_pcie_rx_init_rba(rba); spin_unlock(&rba->lock); spin_lock(&rxq->lock); /* free all first - we might be reconfigured for a different size */ iwl_pcie_rxq_free_rbs(trans); iwl_pcie_rx_init_rxb_lists(rxq); for (i = 0; i < RX_QUEUE_SIZE; i++) rxq->queue[i] = NULL; /* Set us so that we have processed and used all buffers, but have * not restocked the Rx queue with fresh buffers */ rxq->read = rxq->write = 0; rxq->write_actual = 0; memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts)); spin_unlock(&rxq->lock); iwl_pcie_rx_replenish(trans); iwl_pcie_rx_hw_init(trans, rxq); spin_lock(&rxq->lock); iwl_pcie_rxq_inc_wr_ptr(trans); spin_unlock(&rxq->lock); return 0; }
int iwl_pcie_rx_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; int i, err; unsigned long flags; if (!rxq->bd) { err = iwl_pcie_rx_alloc(trans); if (err) return err; } spin_lock_irqsave(&rxq->lock, flags); INIT_LIST_HEAD(&rxq->rx_free); INIT_LIST_HEAD(&rxq->rx_used); INIT_WORK(&trans_pcie->rx_replenish, iwl_pcie_rx_replenish_work); iwl_pcie_rxq_free_rbs(trans); for (i = 0; i < RX_QUEUE_SIZE; i++) rxq->queue[i] = NULL; /* Set us so that we have processed and used all buffers, but have * not restocked the Rx queue with fresh buffers */ rxq->read = rxq->write = 0; rxq->write_actual = 0; rxq->free_count = 0; memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts)); spin_unlock_irqrestore(&rxq->lock, flags); iwl_pcie_rx_replenish(trans); iwl_pcie_rx_hw_init(trans, rxq); spin_lock_irqsave(&trans_pcie->irq_lock, flags); rxq->need_update = 1; iwl_pcie_rxq_inc_wr_ptr(trans, rxq); spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); return 0; }
int iwl_pcie_rx_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; int i, err; if (!rxq->bd) { err = iwl_pcie_rx_alloc(trans); if (err) return err; } spin_lock(&rxq->lock); INIT_WORK(&trans_pcie->rx_replenish, iwl_pcie_rx_replenish_work); /* free all first - we might be reconfigured for a different size */ iwl_pcie_rxq_free_rbs(trans); iwl_pcie_rx_init_rxb_lists(rxq); for (i = 0; i < RX_QUEUE_SIZE; i++) rxq->queue[i] = NULL; /* Set us so that we have processed and used all buffers, but have * not restocked the Rx queue with fresh buffers */ rxq->read = rxq->write = 0; rxq->write_actual = 0; memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts)); spin_unlock(&rxq->lock); iwl_pcie_rx_replenish(trans); iwl_pcie_rx_hw_init(trans, rxq); spin_lock(&trans_pcie->irq_lock); rxq->need_update = 1; iwl_pcie_rxq_inc_wr_ptr(trans, rxq); spin_unlock(&trans_pcie->irq_lock); return 0; }
static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, struct iwl_rx_mem_buffer *rxb) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; bool page_stolen = false; int max_len = PAGE_SIZE << trans_pcie->rx_page_order; u32 offset = 0; if (WARN_ON(!rxb)) return; dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE); while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) { struct iwl_rx_packet *pkt; struct iwl_device_cmd *cmd; u16 sequence; bool reclaim; int index, cmd_index, err, len; struct iwl_rx_cmd_buffer rxcb = { ._offset = offset, ._rx_page_order = trans_pcie->rx_page_order, ._page = rxb->page, ._page_stolen = false, .truesize = max_len, }; pkt = rxb_addr(&rxcb); if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) break; IWL_DEBUG_RX(trans, "cmd at offset %d: %s (0x%.2x)\n", rxcb._offset, get_cmd_string(trans_pcie, pkt->hdr.cmd), pkt->hdr.cmd); len = iwl_rx_packet_len(pkt); len += sizeof(u32); /* account for status word */ trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len); /* Reclaim a command buffer only if this packet is a response * to a (driver-originated) command. * If the packet (e.g. Rx frame) originated from uCode, * there is no command buffer to reclaim. * Ucode should set SEQ_RX_FRAME bit if ucode-originated, * but apparently a few don't get set; catch them here. */ reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME); if (reclaim) { int i; for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) { if (trans_pcie->no_reclaim_cmds[i] == pkt->hdr.cmd) { reclaim = false; break; } } } sequence = le16_to_cpu(pkt->hdr.sequence); index = SEQ_TO_INDEX(sequence); cmd_index = get_cmd_index(&txq->q, index); if (reclaim) cmd = txq->entries[cmd_index].cmd; else cmd = NULL; err = iwl_op_mode_rx(trans->op_mode, &rxcb, cmd); if (reclaim) { kzfree(txq->entries[cmd_index].free_buf); txq->entries[cmd_index].free_buf = NULL; } /* * After here, we should always check rxcb._page_stolen, * if it is true then one of the handlers took the page. */ if (reclaim) { /* Invoke any callbacks, transfer the buffer to caller, * and fire off the (possibly) blocking * iwl_trans_send_cmd() * as we reclaim the driver command queue */ if (!rxcb._page_stolen) iwl_pcie_hcmd_complete(trans, &rxcb, err); else IWL_WARN(trans, "Claim null rxb?\n"); } page_stolen |= rxcb._page_stolen; offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); } /* page was stolen from us -- free our reference */ if (page_stolen) { __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; } /* Reuse the page if possible. For notification packets and * SKBs that fail to Rx correctly, add them back into the * rx_free list for reuse later. */ if (rxb->page != NULL) { rxb->page_dma = dma_map_page(trans->dev, rxb->page, 0, PAGE_SIZE << trans_pcie->rx_page_order, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { /* * free the page(s) as well to not break * the invariant that the items on the used * list have no page(s) */ __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; list_add_tail(&rxb->list, &rxq->rx_used); } else { list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; } } else list_add_tail(&rxb->list, &rxq->rx_used); } /* * iwl_pcie_rx_handle - Main entry function for receiving responses from fw */ static void iwl_pcie_rx_handle(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; u32 r, i; u8 fill_rx = 0; u32 count = 8; int total_empty; restart: spin_lock(&rxq->lock); /* uCode's read index (stored in shared DRAM) indicates the last Rx * buffer that the driver may process (last buffer filled by ucode). */ r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF; i = rxq->read; /* Rx interrupt, but nothing sent from uCode */ if (i == r) IWL_DEBUG_RX(trans, "HW = SW = %d\n", r); /* calculate total frames need to be restock after handling RX */ total_empty = r - rxq->write_actual; if (total_empty < 0) total_empty += RX_QUEUE_SIZE; if (total_empty > (RX_QUEUE_SIZE / 2)) fill_rx = 1; while (i != r) { struct iwl_rx_mem_buffer *rxb; rxb = rxq->queue[i]; rxq->queue[i] = NULL; IWL_DEBUG_RX(trans, "rxbuf: HW = %d, SW = %d (%p)\n", r, i, rxb); iwl_pcie_rx_handle_rb(trans, rxb); i = (i + 1) & RX_QUEUE_MASK; /* If there are a lot of unused frames, * restock the Rx queue so ucode wont assert. */ if (fill_rx) { count++; if (count >= 8) { rxq->read = i; spin_unlock(&rxq->lock); iwl_pcie_rx_replenish(trans, GFP_ATOMIC); count = 0; goto restart; } } } /* Backtrack one entry */ rxq->read = i; spin_unlock(&rxq->lock); if (fill_rx) iwl_pcie_rx_replenish(trans, GFP_ATOMIC); else iwl_pcie_rxq_restock(trans); if (trans_pcie->napi.poll) napi_gro_flush(&trans_pcie->napi, false); } /* * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card */ static void iwl_pcie_irq_handle_error(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); /* W/A for WiFi/WiMAX coex and WiMAX own the RF */ if (trans->cfg->internal_wimax_coex && (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) & APMS_CLK_VAL_MRB_FUNC_MODE) || (iwl_read_prph(trans, APMG_PS_CTRL_REG) & APMG_PS_CTRL_VAL_RESET_REQ))) { clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); iwl_op_mode_wimax_active(trans->op_mode); wake_up(&trans_pcie->wait_command_queue); return; } iwl_pcie_dump_csr(trans); iwl_dump_fh(trans, NULL); local_bh_disable(); /* The STATUS_FW_ERROR bit is set in this function. This must happen * before we wake up the command caller, to ensure a proper cleanup. */ iwl_trans_fw_error(trans); local_bh_enable(); clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); wake_up(&trans_pcie->wait_command_queue); } static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans) { u32 inta; lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock); trace_iwlwifi_dev_irq(trans->dev); /* Discover which interrupts are active/pending */ inta = iwl_read32(trans, CSR_INT); /* the thread will service interrupts and re-enable them */ return inta; }