/** * i40e_write_dword - replace HMC context dword * @hmc_bits: pointer to the HMC memory * @ce_info: a description of the struct to be read from * @src: the struct to be read from **/ static void i40e_write_dword(u8 *hmc_bits, struct i40e_context_ele *ce_info, u8 *src) { u32 src_dword, mask; u8 *from, *dest; u16 shift_width; __le32 dest_dword; /* copy from the next struct field */ from = src + ce_info->offset; /* prepare the bits and mask */ shift_width = ce_info->lsb % 8; /* if the field width is exactly 32 on an x86 machine, then the shift * operation will not work because the SHL instructions count is masked * to 5 bits so the shift will do nothing */ if (ce_info->width < 32) mask = ((u32)1 << ce_info->width) - 1; else mask = ~(u32)0; /* don't swizzle the bits until after the mask because the mask bits * will be in a different bit position on big endian machines */ src_dword = *(u32 *)from; src_dword &= mask; /* shift to correct alignment */ mask <<= shift_width; src_dword <<= shift_width; /* get the current bits from the target bit string */ dest = hmc_bits + (ce_info->lsb / 8); i40e_memcpy(&dest_dword, dest, sizeof(dest_dword), I40E_DMA_TO_NONDMA); dest_dword &= ~(CPU_TO_LE32(mask)); /* get the bits not changing */ dest_dword |= CPU_TO_LE32(src_dword); /* add in the new bits */ /* put it all back */ i40e_memcpy(dest, &dest_dword, sizeof(dest_dword), I40E_NONDMA_TO_DMA); }
/** * i40e_read_word - read HMC context word into struct * @hmc_bits: pointer to the HMC memory * @ce_info: a description of the struct to be filled * @dest: the struct to be filled **/ static void i40e_read_word(u8 *hmc_bits, struct i40e_context_ele *ce_info, u8 *dest) { u16 dest_word, mask; u8 *src, *target; u16 shift_width; __le16 src_word; /* prepare the bits and mask */ shift_width = ce_info->lsb % 8; mask = ((u16)1 << ce_info->width) - 1; /* shift to correct alignment */ mask <<= shift_width; /* get the current bits from the src bit string */ src = hmc_bits + (ce_info->lsb / 8); i40e_memcpy(&src_word, src, sizeof(src_word), I40E_DMA_TO_NONDMA); /* the data in the memory is stored as little endian so mask it * correctly */ src_word &= ~(CPU_TO_LE16(mask)); /* get the data back into host order before shifting */ dest_word = LE16_TO_CPU(src_word); dest_word >>= shift_width; /* get the address from the struct field */ target = dest + ce_info->offset; /* put it back in the struct */ i40e_memcpy(target, &dest_word, sizeof(dest_word), I40E_NONDMA_TO_DMA); }
/** * i40e_write_word - replace HMC context word * @hmc_bits: pointer to the HMC memory * @ce_info: a description of the struct to be read from * @src: the struct to be read from **/ static void i40e_write_word(u8 *hmc_bits, struct i40e_context_ele *ce_info, u8 *src) { u16 src_word, mask; u8 *from, *dest; u16 shift_width; __le16 dest_word; /* copy from the next struct field */ from = src + ce_info->offset; /* prepare the bits and mask */ shift_width = ce_info->lsb % 8; mask = ((u16)1 << ce_info->width) - 1; /* don't swizzle the bits until after the mask because the mask bits * will be in a different bit position on big endian machines */ src_word = *(u16 *)from; src_word &= mask; /* shift to correct alignment */ mask <<= shift_width; src_word <<= shift_width; /* get the current bits from the target bit string */ dest = hmc_bits + (ce_info->lsb / 8); i40e_memcpy(&dest_word, dest, sizeof(dest_word), I40E_DMA_TO_NONDMA); dest_word &= ~(CPU_TO_LE16(mask)); /* get the bits not changing */ dest_word |= CPU_TO_LE16(src_word); /* add in the new bits */ /* put it all back */ i40e_memcpy(dest, &dest_word, sizeof(dest_word), I40E_NONDMA_TO_DMA); }
/** * i40e_asq_send_command - send command to Admin Queue * @hw: pointer to the hw struct * @desc: prefilled descriptor describing the command (non DMA mem) * @buff: buffer to use for indirect commands * @buff_size: size of buffer for indirect commands * @cmd_details: pointer to command details structure * * This is the main send command driver routine for the Admin Queue send * queue. It runs the queue, cleans the queue, etc **/ enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw, struct i40e_aq_desc *desc, void *buff, /* can be NULL */ u16 buff_size, struct i40e_asq_cmd_details *cmd_details) { enum i40e_status_code status = I40E_SUCCESS; struct i40e_dma_mem *dma_buff = NULL; struct i40e_asq_cmd_details *details; struct i40e_aq_desc *desc_on_ring; bool cmd_completed = false; u16 retval = 0; u32 val = 0; i40e_acquire_spinlock(&hw->aq.asq_spinlock); hw->aq.asq_last_status = I40E_AQ_RC_OK; if (hw->aq.asq.count == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Admin queue not initialized.\n"); status = I40E_ERR_QUEUE_EMPTY; goto asq_send_command_error; } val = rd32(hw, hw->aq.asq.head); if (val >= hw->aq.num_asq_entries) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: head overrun at %d\n", val); status = I40E_ERR_QUEUE_EMPTY; goto asq_send_command_error; } details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use); if (cmd_details) { i40e_memcpy(details, cmd_details, sizeof(struct i40e_asq_cmd_details), I40E_NONDMA_TO_NONDMA); /* If the cmd_details are defined copy the cookie. The * CPU_TO_LE32 is not needed here because the data is ignored * by the FW, only used by the driver */ if (details->cookie) { desc->cookie_high = CPU_TO_LE32(I40E_HI_DWORD(details->cookie)); desc->cookie_low = CPU_TO_LE32(I40E_LO_DWORD(details->cookie)); } } else { i40e_memset(details, 0, sizeof(struct i40e_asq_cmd_details), I40E_NONDMA_MEM); } /* clear requested flags and then set additional flags if defined */ desc->flags &= ~CPU_TO_LE16(details->flags_dis); desc->flags |= CPU_TO_LE16(details->flags_ena); if (buff_size > hw->aq.asq_buf_size) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Invalid buffer size: %d.\n", buff_size); status = I40E_ERR_INVALID_SIZE; goto asq_send_command_error; } if (details->postpone && !details->async) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Async flag not set along with postpone flag"); status = I40E_ERR_PARAM; goto asq_send_command_error; } /* call clean and check queue available function to reclaim the * descriptors that were processed by FW, the function returns the * number of desc available */ /* the clean function called here could be called in a separate thread * in case of asynchronous completions */ if (i40e_clean_asq(hw) == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Error queue is full.\n"); status = I40E_ERR_ADMIN_QUEUE_FULL; goto asq_send_command_error; } /* initialize the temp desc pointer with the right desc */ desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use); /* if the desc is available copy the temp desc to the right place */ i40e_memcpy(desc_on_ring, desc, sizeof(struct i40e_aq_desc), I40E_NONDMA_TO_DMA); /* if buff is not NULL assume indirect command */ if (buff != NULL) { dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]); /* copy the user buff into the respective DMA buff */ i40e_memcpy(dma_buff->va, buff, buff_size, I40E_NONDMA_TO_DMA); desc_on_ring->datalen = CPU_TO_LE16(buff_size); /* Update the address values in the desc with the pa value * for respective buffer */ desc_on_ring->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(dma_buff->pa)); desc_on_ring->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(dma_buff->pa)); } /* bump the tail */ i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n"); i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc_on_ring, buff, buff_size); (hw->aq.asq.next_to_use)++; if (hw->aq.asq.next_to_use == hw->aq.asq.count) hw->aq.asq.next_to_use = 0; if (!details->postpone) wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use); /* if cmd_details are not defined or async flag is not set, * we need to wait for desc write back */ if (!details->async && !details->postpone) { u32 total_delay = 0; do { /* AQ designers suggest use of head for better * timing reliability than DD bit */ if (i40e_asq_done(hw)) break; i40e_usec_delay(50); total_delay += 50; } while (total_delay < hw->aq.asq_cmd_timeout); } /* if ready, copy the desc back to temp */ if (i40e_asq_done(hw)) { i40e_memcpy(desc, desc_on_ring, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); if (buff != NULL) i40e_memcpy(buff, dma_buff->va, buff_size, I40E_DMA_TO_NONDMA); retval = LE16_TO_CPU(desc->retval); if (retval != 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Command completed with error 0x%X.\n", retval); /* strip off FW internal code */ retval &= 0xff; } cmd_completed = true; if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK) status = I40E_SUCCESS; else if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_EBUSY) status = I40E_ERR_NOT_READY; else status = I40E_ERR_ADMIN_QUEUE_ERROR; hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval; } i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer writeback:\n"); i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size); /* save writeback aq if requested */ if (details->wb_desc) i40e_memcpy(details->wb_desc, desc_on_ring, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); /* update the error if time out occurred */ if ((!cmd_completed) && (!details->async && !details->postpone)) { #ifdef PF_DRIVER if (rd32(hw, hw->aq.asq.len) & I40E_GL_ATQLEN_ATQCRIT_MASK) { #else if (rd32(hw, hw->aq.asq.len) & I40E_VF_ATQLEN1_ATQCRIT_MASK) { #endif i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: AQ Critical error.\n"); status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR; } else { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Writeback timeout.\n"); status = I40E_ERR_ADMIN_QUEUE_TIMEOUT; } } asq_send_command_error: i40e_release_spinlock(&hw->aq.asq_spinlock); return status; } /** * i40e_fill_default_direct_cmd_desc - AQ descriptor helper function * @desc: pointer to the temp descriptor (non DMA mem) * @opcode: the opcode can be used to decide which flags to turn off or on * * Fill the desc with default values **/ void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc *desc, u16 opcode) { /* zero out the desc */ i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_NONDMA_MEM); desc->opcode = CPU_TO_LE16(opcode); desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_SI); }
/** * i40e_clean_arq_element * @hw: pointer to the hw struct * @e: event info from the receive descriptor, includes any buffers * @pending: number of events that could be left to process * * This function cleans one Admin Receive Queue element and returns * the contents through e. It can also return how many events are * left to process through 'pending' **/ enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw, struct i40e_arq_event_info *e, u16 *pending) { enum i40e_status_code ret_code = I40E_SUCCESS; u16 ntc = hw->aq.arq.next_to_clean; struct i40e_aq_desc *desc; struct i40e_dma_mem *bi; u16 desc_idx; u16 datalen; u16 flags; u16 ntu; /* pre-clean the event info */ i40e_memset(&e->desc, 0, sizeof(e->desc), I40E_NONDMA_MEM); /* take the lock before we start messing with the ring */ i40e_acquire_spinlock(&hw->aq.arq_spinlock); if (hw->aq.arq.count == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Admin queue not initialized.\n"); ret_code = I40E_ERR_QUEUE_EMPTY; goto clean_arq_element_err; } /* set next_to_use to head */ #ifdef INTEGRATED_VF if (!i40e_is_vf(hw)) ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK; else ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK; #else #ifdef PF_DRIVER ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK; #endif /* PF_DRIVER */ #ifdef VF_DRIVER ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK; #endif /* VF_DRIVER */ #endif /* INTEGRATED_VF */ if (ntu == ntc) { /* nothing to do - shouldn't need to update ring's values */ ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK; goto clean_arq_element_out; } /* now clean the next descriptor */ desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc); desc_idx = ntc; hw->aq.arq_last_status = (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval); flags = LE16_TO_CPU(desc->flags); if (flags & I40E_AQ_FLAG_ERR) { ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Event received with error 0x%X.\n", hw->aq.arq_last_status); } i40e_memcpy(&e->desc, desc, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); datalen = LE16_TO_CPU(desc->datalen); e->msg_len = min(datalen, e->buf_len); if (e->msg_buf != NULL && (e->msg_len != 0)) i40e_memcpy(e->msg_buf, hw->aq.arq.r.arq_bi[desc_idx].va, e->msg_len, I40E_DMA_TO_NONDMA); i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n"); i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf, hw->aq.arq_buf_size); /* Restore the original datalen and buffer address in the desc, * FW updates datalen to indicate the event message * size */ bi = &hw->aq.arq.r.arq_bi[ntc]; i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_DMA_MEM); desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); desc->datalen = CPU_TO_LE16((u16)bi->size); desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); /* set tail = the last cleaned desc index. */ wr32(hw, hw->aq.arq.tail, ntc); /* ntc is updated to tail + 1 */ ntc++; if (ntc == hw->aq.num_arq_entries) ntc = 0; hw->aq.arq.next_to_clean = ntc; hw->aq.arq.next_to_use = ntu; #ifdef PF_DRIVER i40e_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode), &e->desc); #endif /* PF_DRIVER */ clean_arq_element_out: /* Set pending if needed, unlock and return */ if (pending != NULL) *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc); clean_arq_element_err: i40e_release_spinlock(&hw->aq.arq_spinlock); return ret_code; }
/** * i40e_add_sd_table_entry - Adds a segment descriptor to the table * @hw: pointer to our hw struct * @hmc_info: pointer to the HMC configuration information struct * @sd_index: segment descriptor index to manipulate * @type: what type of segment descriptor we're manipulating * @direct_mode_sz: size to alloc in direct mode **/ enum i40e_status_code i40e_add_sd_table_entry(struct i40e_hw *hw, struct i40e_hmc_info *hmc_info, u32 sd_index, enum i40e_sd_entry_type type, u64 direct_mode_sz) { enum i40e_status_code ret_code = I40E_SUCCESS; struct i40e_hmc_sd_entry *sd_entry; enum i40e_memory_type mem_type; bool dma_mem_alloc_done = false; struct i40e_dma_mem mem; u64 alloc_len; if (NULL == hmc_info->sd_table.sd_entry) { ret_code = I40E_ERR_BAD_PTR; DEBUGOUT("i40e_add_sd_table_entry: bad sd_entry\n"); goto exit; } if (sd_index >= hmc_info->sd_table.sd_cnt) { ret_code = I40E_ERR_INVALID_SD_INDEX; DEBUGOUT("i40e_add_sd_table_entry: bad sd_index\n"); goto exit; } sd_entry = &hmc_info->sd_table.sd_entry[sd_index]; if (!sd_entry->valid) { if (I40E_SD_TYPE_PAGED == type) { mem_type = i40e_mem_pd; alloc_len = I40E_HMC_PAGED_BP_SIZE; } else { mem_type = i40e_mem_bp_jumbo; alloc_len = direct_mode_sz; } /* allocate a 4K pd page or 2M backing page */ ret_code = i40e_allocate_dma_mem(hw, &mem, mem_type, alloc_len, I40E_HMC_PD_BP_BUF_ALIGNMENT); if (ret_code) goto exit; dma_mem_alloc_done = true; if (I40E_SD_TYPE_PAGED == type) { ret_code = i40e_allocate_virt_mem(hw, &sd_entry->u.pd_table.pd_entry_virt_mem, sizeof(struct i40e_hmc_pd_entry) * 512); if (ret_code) goto exit; sd_entry->u.pd_table.pd_entry = (struct i40e_hmc_pd_entry *) sd_entry->u.pd_table.pd_entry_virt_mem.va; i40e_memcpy(&sd_entry->u.pd_table.pd_page_addr, &mem, sizeof(struct i40e_dma_mem), I40E_NONDMA_TO_NONDMA); } else { i40e_memcpy(&sd_entry->u.bp.addr, &mem, sizeof(struct i40e_dma_mem), I40E_NONDMA_TO_NONDMA); sd_entry->u.bp.sd_pd_index = sd_index; } /* initialize the sd entry */ hmc_info->sd_table.sd_entry[sd_index].entry_type = type; /* increment the ref count */ I40E_INC_SD_REFCNT(&hmc_info->sd_table); } /* Increment backing page reference count */ if (I40E_SD_TYPE_DIRECT == sd_entry->entry_type) I40E_INC_BP_REFCNT(&sd_entry->u.bp); exit: if (I40E_SUCCESS != ret_code) if (dma_mem_alloc_done) i40e_free_dma_mem(hw, &mem); return ret_code; }
/** * i40e_add_pd_table_entry - Adds page descriptor to the specified table * @hw: pointer to our HW structure * @hmc_info: pointer to the HMC configuration information structure * @pd_index: which page descriptor index to manipulate * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one. * * This function: * 1. Initializes the pd entry * 2. Adds pd_entry in the pd_table * 3. Mark the entry valid in i40e_hmc_pd_entry structure * 4. Initializes the pd_entry's ref count to 1 * assumptions: * 1. The memory for pd should be pinned down, physically contiguous and * aligned on 4K boundary and zeroed memory. * 2. It should be 4K in size. **/ enum i40e_status_code i40e_add_pd_table_entry(struct i40e_hw *hw, struct i40e_hmc_info *hmc_info, u32 pd_index, struct i40e_dma_mem *rsrc_pg) { enum i40e_status_code ret_code = I40E_SUCCESS; struct i40e_hmc_pd_table *pd_table; struct i40e_hmc_pd_entry *pd_entry; struct i40e_dma_mem mem; struct i40e_dma_mem *page = &mem; u32 sd_idx, rel_pd_idx; u64 *pd_addr; u64 page_desc; if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) { ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX; DEBUGOUT("i40e_add_pd_table_entry: bad pd_index\n"); goto exit; } /* find corresponding sd */ sd_idx = (pd_index / I40E_HMC_PD_CNT_IN_SD); if (I40E_SD_TYPE_PAGED != hmc_info->sd_table.sd_entry[sd_idx].entry_type) goto exit; rel_pd_idx = (pd_index % I40E_HMC_PD_CNT_IN_SD); pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; pd_entry = &pd_table->pd_entry[rel_pd_idx]; if (!pd_entry->valid) { if (rsrc_pg) { pd_entry->rsrc_pg = true; page = rsrc_pg; } else { /* allocate a 4K backing page */ ret_code = i40e_allocate_dma_mem(hw, page, i40e_mem_bp, I40E_HMC_PAGED_BP_SIZE, I40E_HMC_PD_BP_BUF_ALIGNMENT); if (ret_code) goto exit; pd_entry->rsrc_pg = false; } i40e_memcpy(&pd_entry->bp.addr, page, sizeof(struct i40e_dma_mem), I40E_NONDMA_TO_NONDMA); pd_entry->bp.sd_pd_index = pd_index; pd_entry->bp.entry_type = I40E_SD_TYPE_PAGED; /* Set page address and valid bit */ page_desc = page->pa | 0x1; pd_addr = (u64 *)pd_table->pd_page_addr.va; pd_addr += rel_pd_idx; /* Add the backing page physical address in the pd entry */ i40e_memcpy(pd_addr, &page_desc, sizeof(u64), I40E_NONDMA_TO_DMA); pd_entry->sd_index = sd_idx; pd_entry->valid = true; I40E_INC_PD_REFCNT(pd_table); } I40E_INC_BP_REFCNT(&pd_entry->bp); exit: return ret_code; }
/** * i40e_asq_send_command - send command to Admin Queue * @hw: pointer to the hw struct * @desc: prefilled descriptor describing the command (non DMA mem) * @buff: buffer to use for indirect commands * @buff_size: size of buffer for indirect commands * @cmd_details: pointer to command details structure * * This is the main send command driver routine for the Admin Queue send * queue. It runs the queue, cleans the queue, etc **/ enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw, struct i40e_aq_desc *desc, void *buff, /* can be NULL */ u16 buff_size, struct i40e_asq_cmd_details *cmd_details) { #ifdef I40E_QV struct i40e_aq_desc qv_desc = {0}; struct i40e_aq_desc *qv_desc_on_ring; #endif /* I40E_QV */ enum i40e_status_code status = I40E_SUCCESS; struct i40e_dma_mem *dma_buff = NULL; struct i40e_asq_cmd_details *details; struct i40e_aq_desc *desc_on_ring; bool cmd_completed = FALSE; u16 retval = 0; u32 val = 0; val = rd32(hw, hw->aq.asq.head); if (val >= hw->aq.num_asq_entries) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: head overrun at %d\n", val); status = I40E_ERR_QUEUE_EMPTY; goto asq_send_command_exit; } if (hw->aq.asq.count == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Admin queue not initialized.\n"); status = I40E_ERR_QUEUE_EMPTY; goto asq_send_command_exit; } if (i40e_is_nvm_update_op(desc) && hw->aq.nvm_busy) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: NVM busy.\n"); status = I40E_ERR_NVM; goto asq_send_command_exit; } details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use); if (cmd_details) { i40e_memcpy(details, cmd_details, sizeof(struct i40e_asq_cmd_details), I40E_NONDMA_TO_NONDMA); /* If the cmd_details are defined copy the cookie. The * CPU_TO_LE32 is not needed here because the data is ignored * by the FW, only used by the driver */ if (details->cookie) { desc->cookie_high = CPU_TO_LE32(I40E_HI_DWORD(details->cookie)); desc->cookie_low = CPU_TO_LE32(I40E_LO_DWORD(details->cookie)); } } else { i40e_memset(details, 0, sizeof(struct i40e_asq_cmd_details), I40E_NONDMA_MEM); } /* clear requested flags and then set additional flags if defined */ desc->flags &= ~CPU_TO_LE16(details->flags_dis); desc->flags |= CPU_TO_LE16(details->flags_ena); i40e_acquire_spinlock(&hw->aq.asq_spinlock); if (buff_size > hw->aq.asq_buf_size) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Invalid buffer size: %d.\n", buff_size); status = I40E_ERR_INVALID_SIZE; goto asq_send_command_error; } if (details->postpone && !details->async) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Async flag not set along with postpone flag"); status = I40E_ERR_PARAM; goto asq_send_command_error; } /* call clean and check queue available function to reclaim the * descriptors that were processed by FW, the function returns the * number of desc available */ /* the clean function called here could be called in a separate thread * in case of asynchronous completions */ if (i40e_clean_asq(hw) == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Error queue is full.\n"); status = I40E_ERR_ADMIN_QUEUE_FULL; goto asq_send_command_error; } /* initialize the temp desc pointer with the right desc */ desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use); /* if the desc is available copy the temp desc to the right place */ i40e_memcpy(desc_on_ring, desc, sizeof(struct i40e_aq_desc), I40E_NONDMA_TO_DMA); #ifdef I40E_QV /* copy the descriptor from ring to userspace buffer */ i40e_memcpy(&qv_desc, desc_on_ring, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); qv_desc_on_ring = desc_on_ring; desc_on_ring = &qv_desc; #endif /* I40E_QV */ /* if buff is not NULL assume indirect command */ if (buff != NULL) { dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]); /* copy the user buff into the respective DMA buff */ i40e_memcpy(dma_buff->va, buff, buff_size, I40E_NONDMA_TO_DMA); desc_on_ring->datalen = CPU_TO_LE16(buff_size); /* Update the address values in the desc with the pa value * for respective buffer */ desc_on_ring->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(dma_buff->pa)); desc_on_ring->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(dma_buff->pa)); #ifdef I40E_QV /* copy the descriptor from userspace buffer to ring */ i40e_memcpy(qv_desc_on_ring, desc_on_ring, sizeof(struct i40e_aq_desc), I40E_NONDMA_TO_DMA); #endif /* I40E_QV */ } /* bump the tail */ i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n"); i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc_on_ring, buff, buff_size); (hw->aq.asq.next_to_use)++; if (hw->aq.asq.next_to_use == hw->aq.asq.count) hw->aq.asq.next_to_use = 0; if (!details->postpone) wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use); /* if cmd_details are not defined or async flag is not set, * we need to wait for desc write back */ if (!details->async && !details->postpone) { u32 total_delay = 0; u32 delay_len = 10; do { #ifdef I40E_QV /* copy the descriptor from ring to user buffer */ i40e_memcpy(desc_on_ring, qv_desc_on_ring, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); #endif /* I40E_QV */ /* AQ designers suggest use of head for better * timing reliability than DD bit */ if (i40e_asq_done(hw)) break; /* ugh! delay while spin_lock */ i40e_usec_delay(delay_len); total_delay += delay_len; } while (total_delay < hw->aq.asq_cmd_timeout); } /* if ready, copy the desc back to temp */ if (i40e_asq_done(hw)) { #ifdef I40E_QV /* Swap pointer back */ desc_on_ring = qv_desc_on_ring; #endif /* I40E_QV */ i40e_memcpy(desc, desc_on_ring, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); if (buff != NULL) i40e_memcpy(buff, dma_buff->va, buff_size, I40E_DMA_TO_NONDMA); retval = LE16_TO_CPU(desc->retval); if (retval != 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Command completed with error 0x%X.\n", retval); /* strip off FW internal code */ retval &= 0xff; } cmd_completed = TRUE; if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK) status = I40E_SUCCESS; else status = I40E_ERR_ADMIN_QUEUE_ERROR; hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval; } i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer writeback:\n"); i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size); /* update the error if time out occurred */ if ((!cmd_completed) && (!details->async && !details->postpone)) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Writeback timeout.\n"); status = I40E_ERR_ADMIN_QUEUE_TIMEOUT; } if (!status && i40e_is_nvm_update_op(desc)) hw->aq.nvm_busy = TRUE; asq_send_command_error: i40e_release_spinlock(&hw->aq.asq_spinlock); asq_send_command_exit: return status; }
/** * i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue * @hw: pointer to the hardware structure **/ static enum i40e_status_code i40e_alloc_arq_bufs(struct i40e_hw *hw) { #ifdef I40E_QV struct i40e_aq_desc qv_desc; struct i40e_aq_desc *qv_desc_on_ring; #endif enum i40e_status_code ret_code; struct i40e_aq_desc *desc; struct i40e_dma_mem *bi; int i; /* We'll be allocating the buffer info memory first, then we can * allocate the mapped buffers for the event processing */ /* buffer_info structures do not need alignment */ ret_code = i40e_allocate_virt_mem(hw, &hw->aq.arq.dma_head, (hw->aq.num_arq_entries * sizeof(struct i40e_dma_mem))); if (ret_code) goto alloc_arq_bufs; hw->aq.arq.r.arq_bi = (struct i40e_dma_mem *)hw->aq.arq.dma_head.va; /* allocate the mapped buffers */ for (i = 0; i < hw->aq.num_arq_entries; i++) { bi = &hw->aq.arq.r.arq_bi[i]; ret_code = i40e_allocate_dma_mem(hw, bi, i40e_mem_arq_buf, hw->aq.arq_buf_size, I40E_ADMINQ_DESC_ALIGNMENT); if (ret_code) goto unwind_alloc_arq_bufs; /* now configure the descriptors for use */ desc = I40E_ADMINQ_DESC(hw->aq.arq, i); #ifdef I40E_QV /* swap the descriptor with userspace version */ i40e_memcpy(&qv_desc, desc, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); qv_desc_on_ring = desc; desc = &qv_desc; #endif desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); desc->opcode = 0; /* This is in accordance with Admin queue design, there is no * register for buffer size configuration */ desc->datalen = CPU_TO_LE16((u16)bi->size); desc->retval = 0; desc->cookie_high = 0; desc->cookie_low = 0; desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); desc->params.external.param0 = 0; desc->params.external.param1 = 0; #ifdef I40E_QV /* put the initialized descriptor back to the ring */ i40e_memcpy(qv_desc_on_ring, desc, sizeof(struct i40e_aq_desc), I40E_NONDMA_TO_DMA); #endif } alloc_arq_bufs: return ret_code; unwind_alloc_arq_bufs: /* don't try to free the one that failed... */ i--; for (; i >= 0; i--) i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]); i40e_free_virt_mem(hw, &hw->aq.arq.dma_head); return ret_code; }
/** * i40e_clean_arq_element * @hw: pointer to the hw struct * @e: event info from the receive descriptor, includes any buffers * @pending: number of events that could be left to process * * This function cleans one Admin Receive Queue element and returns * the contents through e. It can also return how many events are * left to process through 'pending' **/ enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw, struct i40e_arq_event_info *e, u16 *pending) { #ifdef I40E_QV struct i40e_aq_desc qv_desc = {0}; struct i40e_aq_desc *qv_desc_on_ring; #endif /* I40E_QV */ enum i40e_status_code ret_code = I40E_SUCCESS; u16 ntc = hw->aq.arq.next_to_clean; struct i40e_aq_desc *desc; struct i40e_dma_mem *bi; u16 desc_idx; u16 datalen; u16 flags; u16 ntu; /* take the lock before we start messing with the ring */ i40e_acquire_spinlock(&hw->aq.arq_spinlock); /* set next_to_use to head */ ntu = (rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK); if (ntu == ntc) { /* nothing to do - shouldn't need to update ring's values */ i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Queue is empty.\n"); ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK; goto clean_arq_element_out; } /* now clean the next descriptor */ desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc); #ifdef I40E_QV /* copy the descriptor from ring to userspace buffer */ i40e_memcpy(&qv_desc, desc, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); qv_desc_on_ring = desc; desc = &qv_desc; #endif /* I40E_QV */ desc_idx = ntc; flags = LE16_TO_CPU(desc->flags); if (flags & I40E_AQ_FLAG_ERR) { ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; hw->aq.arq_last_status = (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval); i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Event received with error 0x%X.\n", hw->aq.arq_last_status); } i40e_memcpy(&e->desc, desc, sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); datalen = LE16_TO_CPU(desc->datalen); e->msg_size = min(datalen, e->msg_size); if (e->msg_buf != NULL && (e->msg_size != 0)) i40e_memcpy(e->msg_buf, hw->aq.arq.r.arq_bi[desc_idx].va, e->msg_size, I40E_DMA_TO_NONDMA); i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n"); i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf, hw->aq.arq_buf_size); /* Restore the original datalen and buffer address in the desc, * FW updates datalen to indicate the event message * size */ bi = &hw->aq.arq.r.arq_bi[ntc]; i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_DMA_MEM); desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); desc->datalen = CPU_TO_LE16((u16)bi->size); desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); #ifdef I40E_QV /* copy the descriptor from userspace buffer to ring */ i40e_memcpy(qv_desc_on_ring, desc, sizeof(struct i40e_aq_desc), I40E_NONDMA_TO_DMA); #endif /* I40E_QV */ /* set tail = the last cleaned desc index. */ wr32(hw, hw->aq.arq.tail, ntc); /* ntc is updated to tail + 1 */ ntc++; if (ntc == hw->aq.num_arq_entries) ntc = 0; hw->aq.arq.next_to_clean = ntc; hw->aq.arq.next_to_use = ntu; clean_arq_element_out: /* Set pending if needed, unlock and return */ if (pending != NULL) *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc); i40e_release_spinlock(&hw->aq.arq_spinlock); if (i40e_is_nvm_update_op(&e->desc)) { hw->aq.nvm_busy = FALSE; if (hw->aq.nvm_release_on_done) { i40e_release_nvm(hw); hw->aq.nvm_release_on_done = FALSE; } } return ret_code; }