void inc_count(void) { ACCESS_ONCE(__get_thread_var(counter))++; }
static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, struct iwl_rx_mem_buffer *rxb) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; bool page_stolen = false; int max_len = PAGE_SIZE << trans_pcie->rx_page_order; u32 offset = 0; if (WARN_ON(!rxb)) return; dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE); while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) { struct iwl_rx_packet *pkt; struct iwl_device_cmd *cmd; u16 sequence; bool reclaim; int index, cmd_index, err, len; struct iwl_rx_cmd_buffer rxcb = { ._offset = offset, ._rx_page_order = trans_pcie->rx_page_order, ._page = rxb->page, ._page_stolen = false, .truesize = max_len, }; pkt = rxb_addr(&rxcb); if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) break; IWL_DEBUG_RX(trans, "cmd at offset %d: %s (0x%.2x)\n", rxcb._offset, get_cmd_string(trans_pcie, pkt->hdr.cmd), pkt->hdr.cmd); len = iwl_rx_packet_len(pkt); len += sizeof(u32); /* account for status word */ trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len); /* Reclaim a command buffer only if this packet is a response * to a (driver-originated) command. * If the packet (e.g. Rx frame) originated from uCode, * there is no command buffer to reclaim. * Ucode should set SEQ_RX_FRAME bit if ucode-originated, * but apparently a few don't get set; catch them here. */ reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME); if (reclaim) { int i; for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) { if (trans_pcie->no_reclaim_cmds[i] == pkt->hdr.cmd) { reclaim = false; break; } } } sequence = le16_to_cpu(pkt->hdr.sequence); index = SEQ_TO_INDEX(sequence); cmd_index = get_cmd_index(&txq->q, index); if (reclaim) cmd = txq->entries[cmd_index].cmd; else cmd = NULL; err = iwl_op_mode_rx(trans->op_mode, &rxcb, cmd); if (reclaim) { kfree(txq->entries[cmd_index].free_buf); txq->entries[cmd_index].free_buf = NULL; } /* * After here, we should always check rxcb._page_stolen, * if it is true then one of the handlers took the page. */ if (reclaim) { /* Invoke any callbacks, transfer the buffer to caller, * and fire off the (possibly) blocking * iwl_trans_send_cmd() * as we reclaim the driver command queue */ if (!rxcb._page_stolen) iwl_pcie_hcmd_complete(trans, &rxcb, err); else IWL_WARN(trans, "Claim null rxb?\n"); } page_stolen |= rxcb._page_stolen; offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); } /* page was stolen from us -- free our reference */ if (page_stolen) { __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; } /* Reuse the page if possible. For notification packets and * SKBs that fail to Rx correctly, add them back into the * rx_free list for reuse later. */ spin_lock(&rxq->lock); if (rxb->page != NULL) { rxb->page_dma = dma_map_page(trans->dev, rxb->page, 0, PAGE_SIZE << trans_pcie->rx_page_order, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { /* * free the page(s) as well to not break * the invariant that the items on the used * list have no page(s) */ __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; list_add_tail(&rxb->list, &rxq->rx_used); } else { list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; } } else list_add_tail(&rxb->list, &rxq->rx_used); spin_unlock(&rxq->lock); } /* * iwl_pcie_rx_handle - Main entry function for receiving responses from fw */ static void iwl_pcie_rx_handle(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; u32 r, i; u8 fill_rx = 0; u32 count = 8; int total_empty; /* uCode's read index (stored in shared DRAM) indicates the last Rx * buffer that the driver may process (last buffer filled by ucode). */ r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF; i = rxq->read; /* Rx interrupt, but nothing sent from uCode */ if (i == r) IWL_DEBUG_RX(trans, "HW = SW = %d\n", r); /* calculate total frames need to be restock after handling RX */ total_empty = r - rxq->write_actual; if (total_empty < 0) total_empty += RX_QUEUE_SIZE; if (total_empty > (RX_QUEUE_SIZE / 2)) fill_rx = 1; while (i != r) { struct iwl_rx_mem_buffer *rxb; rxb = rxq->queue[i]; rxq->queue[i] = NULL; IWL_DEBUG_RX(trans, "rxbuf: HW = %d, SW = %d (%p)\n", r, i, rxb); iwl_pcie_rx_handle_rb(trans, rxb); i = (i + 1) & RX_QUEUE_MASK; /* If there are a lot of unused frames, * restock the Rx queue so ucode wont assert. */ if (fill_rx) { count++; if (count >= 8) { rxq->read = i; iwl_pcie_rx_replenish_now(trans); count = 0; } } } /* Backtrack one entry */ rxq->read = i; if (fill_rx) iwl_pcie_rx_replenish_now(trans); else iwl_pcie_rxq_restock(trans); } /* * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card */ static void iwl_pcie_irq_handle_error(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); /* W/A for WiFi/WiMAX coex and WiMAX own the RF */ if (trans->cfg->internal_wimax_coex && (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) & APMS_CLK_VAL_MRB_FUNC_MODE) || (iwl_read_prph(trans, APMG_PS_CTRL_REG) & APMG_PS_CTRL_VAL_RESET_REQ))) { clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); iwl_op_mode_wimax_active(trans->op_mode); wake_up(&trans_pcie->wait_command_queue); return; } iwl_pcie_dump_csr(trans); iwl_dump_fh(trans, NULL); local_bh_disable(); /* The STATUS_FW_ERROR bit is set in this function. This must happen * before we wake up the command caller, to ensure a proper cleanup. */ iwl_trans_fw_error(trans); local_bh_enable(); clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); wake_up(&trans_pcie->wait_command_queue); } static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans) { u32 inta; lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock); trace_iwlwifi_dev_irq(trans->dev); /* Discover which interrupts are active/pending */ inta = iwl_read32(trans, CSR_INT); /* the thread will service interrupts and re-enable them */ return inta; }
/* * Lock a mutex (possibly interruptible), slowpath: */ static inline int __sched __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, struct lockdep_map *nest_lock, unsigned long ip) { struct task_struct *task = current; struct mutex_waiter waiter; unsigned long flags; preempt_disable(); mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); #ifdef CONFIG_MUTEX_SPIN_ON_OWNER /* * Optimistic spinning. * * We try to spin for acquisition when we find that there are no * pending waiters and the lock owner is currently running on a * (different) CPU. * * The rationale is that if the lock owner is running, it is likely to * release the lock soon. * * Since this needs the lock owner, and this mutex implementation * doesn't track the owner atomically in the lock field, we need to * track it non-atomically. * * We can't do this for DEBUG_MUTEXES because that relies on wait_lock * to serialize everything. * * The mutex spinners are queued up using MCS lock so that only one * spinner can compete for the mutex. However, if mutex spinning isn't * going to happen, there is no point in going through the lock/unlock * overhead. */ if (!mutex_can_spin_on_owner(lock)) goto slowpath; for (;;) { struct task_struct *owner; struct mspin_node node; /* * If there's an owner, wait for it to either * release the lock or go to sleep. */ mspin_lock(MLOCK(lock), &node); owner = ACCESS_ONCE(lock->owner); if (owner && !mutex_spin_on_owner(lock, owner)) { mspin_unlock(MLOCK(lock), &node); break; } if ((atomic_read(&lock->count) == 1) && (atomic_cmpxchg(&lock->count, 1, 0) == 1)) { lock_acquired(&lock->dep_map, ip); mutex_set_owner(lock); mspin_unlock(MLOCK(lock), &node); preempt_enable(); return 0; } mspin_unlock(MLOCK(lock), &node); /* * When there's no owner, we might have preempted between the * owner acquiring the lock and setting the owner field. If * we're an RT task that will live-lock because we won't let * the owner complete. */ if (!owner && (need_resched() || rt_task(task) || is_realtime(task))) break; /* * The cpu_relax() call is a compiler barrier which forces * everything in this loop to be re-loaded. We don't need * memory barriers as we'll eventually observe the right * values at the cost of a few extra spins. */ arch_mutex_cpu_relax(); } slowpath: #endif spin_lock_mutex(&lock->wait_lock, flags); debug_mutex_lock_common(lock, &waiter); debug_mutex_add_waiter(lock, &waiter, task_thread_info(task)); /* add waiting tasks to the end of the waitqueue (FIFO): */ list_add_tail(&waiter.list, &lock->wait_list); waiter.task = task; if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1)) goto done; lock_contended(&lock->dep_map, ip); for (;;) { /* * Lets try to take the lock again - this is needed even if * we get here for the first time (shortly after failing to * acquire the lock), to make sure that we get a wakeup once * it's unlocked. Later on, if we sleep, this is the * operation that gives us the lock. We xchg it to -1, so * that when we release the lock, we properly wake up the * other waiters: */ if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1)) break; /* * got a signal? (This code gets eliminated in the * TASK_UNINTERRUPTIBLE case.) */ if (unlikely(signal_pending_state(state, task))) { mutex_remove_waiter(lock, &waiter, task_thread_info(task)); mutex_release(&lock->dep_map, 1, ip); spin_unlock_mutex(&lock->wait_lock, flags); debug_mutex_free_waiter(&waiter); preempt_enable(); return -EINTR; } __set_task_state(task, state); /* didn't get the lock, go to sleep: */ spin_unlock_mutex(&lock->wait_lock, flags); schedule_preempt_disabled(); spin_lock_mutex(&lock->wait_lock, flags); } done: lock_acquired(&lock->dep_map, ip); /* got the lock - rejoice! */ mutex_remove_waiter(lock, &waiter, current_thread_info()); mutex_set_owner(lock); /* set it to 0 if there are no waiters left: */ if (likely(list_empty(&lock->wait_list))) atomic_set(&lock->count, 0); spin_unlock_mutex(&lock->wait_lock, flags); debug_mutex_free_waiter(&waiter); preempt_enable(); return 0; }
/* * Child maze-solver process. Each child maintains its own ->visited[] * array, and the maze data structure is used to handle contention for * the same cell. Such contention indicates that a pair of child threads * has found a path to each other. */ void *maze_solve_child(void *arg) { int cr; int cc; int nr; int nc; cpu_set_t mask; int vi = 0; struct maze_child *mcp = (struct maze_child *)arg; struct maze_child_shared *mcsp = mcp->mcsp; struct maze *mp = mcsp->mp; static const struct timespec ts = { .tv_sec = 0, .tv_nsec = 3000 }; unsigned long long tstart = current_time(); mymcp = mcp; myid = mcp->myid; CPU_ZERO(&mask); CPU_SET(myid, &mask); sched_setaffinity(0, sizeof(mask), &mask); cr = mcp->visited[vi].row; cc = mcp->visited[vi].col; do { /* * Each pass through the following loop checks one cell * that has already been visited. When a cell is found * that has at least one remaining unexplored neighbor, * go to the following loop to do the exploration. */ while (!maze_find_any_next_cell(mp, cr, cc, &nr, &nc)) { if (++vi >= mcp->vi || ACCESS_ONCE(mcsp->done)) goto done; cr = mcp->visited[vi].row; cc = mcp->visited[vi].col; } /* * Each pass through the following loop attempts to extend * the current path one cell. */ do { if (ACCESS_ONCE(mcsp->done)) goto done; cr = nr; cc = nc; } while (maze_find_any_next_cell(mp, cr, cc, &nr, &nc)); /* * Reset back to the start of this path in case there is * another unexplored cell adjacent to it. */ cr = mcp->visited[vi].row; cc = mcp->visited[vi].col; } while (!maze_solve_child_done_check()); done: /* Capture time, and act as passthrough for visibility information. */ mcp->dt = current_time() - tstart; if (nthreads > 2) while (!maze_solve_child_done_check()) nanosleep(&ts, NULL); /* In case hardware bus-locks. */ return NULL; } /* * The thread encounters are stored, but only by the thread that discovered * the encounter. Copy any encounters to the location that the other * thread would have recorded them. */ void maze_solve_map_adj(struct maze *mp) { int i; int j; struct maze_child *mcp = mp->msp->mcp; for (i = 0; i < nthreads; i++) for (j = 0; j < nthreads; j++) if (mcp[i].adj[j].mr == -1) { mcp[i].adj[j].mr = mcp[j].adj[i].tr; mcp[i].adj[j].mc = mcp[j].adj[i].tc; mcp[i].adj[j].tr = mcp[j].adj[i].mr; mcp[i].adj[j].tc = mcp[j].adj[i].mc; } }
int fscrypt_get_crypt_info(struct inode *inode) { struct fscrypt_info *crypt_info; struct fscrypt_context ctx; struct crypto_skcipher *ctfm; const char *cipher_str; int keysize; u8 *raw_key = NULL; int res; res = fscrypt_initialize(inode->i_sb->s_cop->flags); if (res) return res; if (!inode->i_sb->s_cop->get_context) return -EOPNOTSUPP; retry: crypt_info = ACCESS_ONCE(inode->i_crypt_info); if (crypt_info) { if (!crypt_info->ci_keyring_key || key_validate(crypt_info->ci_keyring_key) == 0) return 0; fscrypt_put_encryption_info(inode, crypt_info); goto retry; } res = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx)); if (res < 0) { if (!fscrypt_dummy_context_enabled(inode)) return res; ctx.format = FS_ENCRYPTION_CONTEXT_FORMAT_V1; ctx.contents_encryption_mode = FS_ENCRYPTION_MODE_AES_256_XTS; ctx.filenames_encryption_mode = FS_ENCRYPTION_MODE_AES_256_CTS; ctx.flags = 0; } else if (res != sizeof(ctx)) { return -EINVAL; } if (ctx.format != FS_ENCRYPTION_CONTEXT_FORMAT_V1) return -EINVAL; if (ctx.flags & ~FS_POLICY_FLAGS_VALID) return -EINVAL; crypt_info = kmem_cache_alloc(fscrypt_info_cachep, GFP_NOFS); if (!crypt_info) return -ENOMEM; crypt_info->ci_flags = ctx.flags; crypt_info->ci_data_mode = ctx.contents_encryption_mode; crypt_info->ci_filename_mode = ctx.filenames_encryption_mode; crypt_info->ci_ctfm = NULL; crypt_info->ci_keyring_key = NULL; memcpy(crypt_info->ci_master_key, ctx.master_key_descriptor, sizeof(crypt_info->ci_master_key)); res = determine_cipher_type(crypt_info, inode, &cipher_str, &keysize); if (res) goto out; /* * This cannot be a stack buffer because it is passed to the scatterlist * crypto API as part of key derivation. */ res = -ENOMEM; raw_key = kmalloc(FS_MAX_KEY_SIZE, GFP_NOFS); if (!raw_key) goto out; if (fscrypt_dummy_context_enabled(inode)) { memset(raw_key, 0x42, keysize/2); memset(raw_key+keysize/2, 0x24, keysize - (keysize/2)); goto got_key; } res = validate_user_key(crypt_info, &ctx, raw_key, FS_KEY_DESC_PREFIX, FS_KEY_DESC_PREFIX_SIZE); if (res && inode->i_sb->s_cop->key_prefix) { u8 *prefix = NULL; int prefix_size, res2; prefix_size = inode->i_sb->s_cop->key_prefix(inode, &prefix); res2 = validate_user_key(crypt_info, &ctx, raw_key, prefix, prefix_size); if (res2) { if (res2 == -ENOKEY) res = -ENOKEY; goto out; } } else if (res) { goto out; } got_key: ctfm = crypto_alloc_skcipher(cipher_str, 0, 0); if (!ctfm || IS_ERR(ctfm)) { res = ctfm ? PTR_ERR(ctfm) : -ENOMEM; printk(KERN_DEBUG "%s: error %d (inode %u) allocating crypto tfm\n", __func__, res, (unsigned) inode->i_ino); goto out; } crypt_info->ci_ctfm = ctfm; crypto_skcipher_clear_flags(ctfm, ~0); crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_REQ_WEAK_KEY); res = crypto_skcipher_setkey(ctfm, raw_key, keysize); if (res) goto out; kzfree(raw_key); raw_key = NULL; if (cmpxchg(&inode->i_crypt_info, NULL, crypt_info) != NULL) { put_crypt_info(crypt_info); goto retry; } return 0; out: if (res == -ENOKEY) res = 0; put_crypt_info(crypt_info); kzfree(raw_key); return res; }
static struct dentry *ovl_upperdentry_dereference(struct ovl_entry *oe) { struct dentry *upperdentry = ACCESS_ONCE(oe->__upperdentry); smp_read_barrier_depends(); return upperdentry; }
int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, pmd_t *pmdp, unsigned long trap, int local, int ssize, unsigned int psize) { unsigned int index, valid; unsigned char *hpte_slot_array; unsigned long rflags, pa, hidx; unsigned long old_pmd, new_pmd; int ret, lpsize = MMU_PAGE_16M; unsigned long vpn, hash, shift, slot; /* * atomically mark the linux large page PMD busy and dirty */ do { pmd_t pmd = ACCESS_ONCE(*pmdp); old_pmd = pmd_val(pmd); /* If PMD busy, retry the access */ if (unlikely(old_pmd & _PAGE_BUSY)) return 0; /* If PMD is trans splitting retry the access */ if (unlikely(old_pmd & _PAGE_SPLITTING)) return 0; /* If PMD permissions don't match, take page fault */ if (unlikely(access & ~old_pmd)) return 1; /* * Try to lock the PTE, add ACCESSED and DIRTY if it was * a write access */ new_pmd = old_pmd | _PAGE_BUSY | _PAGE_ACCESSED; if (access & _PAGE_RW) new_pmd |= _PAGE_DIRTY; } while (old_pmd != __cmpxchg_u64((unsigned long *)pmdp, old_pmd, new_pmd)); /* * PP bits. _PAGE_USER is already PP bit 0x2, so we only * need to add in 0x1 if it's a read-only user page */ rflags = new_pmd & _PAGE_USER; if ((new_pmd & _PAGE_USER) && !((new_pmd & _PAGE_RW) && (new_pmd & _PAGE_DIRTY))) rflags |= 0x1; /* * _PAGE_EXEC -> HW_NO_EXEC since it's inverted */ rflags |= ((new_pmd & _PAGE_EXEC) ? 0 : HPTE_R_N); #if 0 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { /* * No CPU has hugepages but lacks no execute, so we * don't need to worry about that case */ rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap); } #endif /* * Find the slot index details for this ea, using base page size. */ shift = mmu_psize_defs[psize].shift; index = (ea & ~HPAGE_PMD_MASK) >> shift; BUG_ON(index >= 4096); vpn = hpt_vpn(ea, vsid, ssize); hash = hpt_hash(vpn, shift, ssize); hpte_slot_array = get_hpte_slot_array(pmdp); if (psize == MMU_PAGE_4K) { /* * invalidate the old hpte entry if we have that mapped via 64K * base page size. This is because demote_segment won't flush * hash page table entries. */ if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO)) invalidate_old_hpte(vsid, ea, pmdp, MMU_PAGE_64K, ssize); } valid = hpte_valid(hpte_slot_array, index); if (valid) { /* update the hpte bits */ hidx = hpte_hash_index(hpte_slot_array, index); if (hidx & _PTEIDX_SECONDARY) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; slot += hidx & _PTEIDX_GROUP_IX; ret = ppc_md.hpte_updatepp(slot, rflags, vpn, psize, lpsize, ssize, local); /* * We failed to update, try to insert a new entry. */ if (ret == -1) { /* * large pte is marked busy, so we can be sure * nobody is looking at hpte_slot_array. hence we can * safely update this here. */ valid = 0; hpte_slot_array[index] = 0; } } if (!valid) { unsigned long hpte_group; /* insert new entry */ pa = pmd_pfn(__pmd(old_pmd)) << PAGE_SHIFT; new_pmd |= _PAGE_HASHPTE; /* Add in WIMG bits */ rflags |= (new_pmd & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | _PAGE_GUARDED)); /* * enable the memory coherence always */ rflags |= HPTE_R_M; repeat: hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; /* Insert into the hash table, primary slot */ slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, psize, lpsize, ssize); /* * Primary is full, try the secondary */ if (unlikely(slot == -1)) { hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, HPTE_V_SECONDARY, psize, lpsize, ssize); if (slot == -1) { if (mftb() & 0x1) hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; ppc_md.hpte_remove(hpte_group); goto repeat; } } /* * Hypervisor failure. Restore old pmd and return -1 * similar to __hash_page_* */ if (unlikely(slot == -2)) { *pmdp = __pmd(old_pmd); hash_failure_debug(ea, access, vsid, trap, ssize, psize, lpsize, old_pmd); return -1; } /* * large pte is marked busy, so we can be sure * nobody is looking at hpte_slot_array. hence we can * safely update this here. */ mark_hpte_slot_valid(hpte_slot_array, index, slot); } /* * Mark the pte with _PAGE_COMBO, if we are trying to hash it with * base page size 4k. */ if (psize == MMU_PAGE_4K) new_pmd |= _PAGE_COMBO; /* * The hpte valid is stored in the pgtable whose address is in the * second half of the PMD. Order this against clearing of the busy bit in * huge pmd. */ smp_wmb(); *pmdp = __pmd(new_pmd & ~_PAGE_BUSY); return 0; }
static int ar9002_hw_proc_txdesc(struct ath_hw *ah, void *ds, struct ath_tx_status *ts) { struct ar5416_desc *ads = AR5416DESC(ds); u32 status; //printk("Daekyeong: ar9002_hw_proc_txdesc()\n"); status = ACCESS_ONCE(ads->ds_txstatus9); if ((status & AR_TxDone) == 0) return -EINPROGRESS; ts->ts_tstamp = ads->AR_SendTimestamp; ts->ts_status = 0; ts->ts_flags = 0; if (status & AR_TxOpExceeded) ts->ts_status |= ATH9K_TXERR_XTXOP; ts->tid = MS(status, AR_TxTid); ts->ts_rateindex = MS(status, AR_FinalTxIdx); ts->ts_seqnum = MS(status, AR_SeqNum); status = ACCESS_ONCE(ads->ds_txstatus0); ts->ts_rssi_ctl0 = MS(status, AR_TxRSSIAnt00); ts->ts_rssi_ctl1 = MS(status, AR_TxRSSIAnt01); ts->ts_rssi_ctl2 = MS(status, AR_TxRSSIAnt02); if (status & AR_TxBaStatus) { ts->ts_flags |= ATH9K_TX_BA; ts->ba_low = ads->AR_BaBitmapLow; ts->ba_high = ads->AR_BaBitmapHigh; } status = ACCESS_ONCE(ads->ds_txstatus1); if (status & AR_FrmXmitOK) ts->ts_status |= ATH9K_TX_ACKED; else { if (status & AR_ExcessiveRetries) ts->ts_status |= ATH9K_TXERR_XRETRY; if (status & AR_Filtered) ts->ts_status |= ATH9K_TXERR_FILT; if (status & AR_FIFOUnderrun) { ts->ts_status |= ATH9K_TXERR_FIFO; ath9k_hw_updatetxtriglevel(ah, true); } } if (status & AR_TxTimerExpired) ts->ts_status |= ATH9K_TXERR_TIMER_EXPIRED; if (status & AR_DescCfgErr) ts->ts_flags |= ATH9K_TX_DESC_CFG_ERR; if (status & AR_TxDataUnderrun) { ts->ts_flags |= ATH9K_TX_DATA_UNDERRUN; ath9k_hw_updatetxtriglevel(ah, true); } if (status & AR_TxDelimUnderrun) { ts->ts_flags |= ATH9K_TX_DELIM_UNDERRUN; ath9k_hw_updatetxtriglevel(ah, true); } ts->ts_shortretry = MS(status, AR_RTSFailCnt); ts->ts_longretry = MS(status, AR_DataFailCnt); ts->ts_virtcol = MS(status, AR_VirtRetryCnt); status = ACCESS_ONCE(ads->ds_txstatus5); ts->ts_rssi = MS(status, AR_TxRSSICombined); ts->ts_rssi_ext0 = MS(status, AR_TxRSSIAnt10); ts->ts_rssi_ext1 = MS(status, AR_TxRSSIAnt11); ts->ts_rssi_ext2 = MS(status, AR_TxRSSIAnt12); ts->evm0 = ads->AR_TxEVM0; ts->evm1 = ads->AR_TxEVM1; ts->evm2 = ads->AR_TxEVM2; return 0; }
static inline void pdma_write_skb_fragment(END_DEVICE *ei_local, dma_addr_t frag_addr, u32 frag_size, u32 *desc_odd, u32 *txd_info2, const u32 txd_info4, struct sk_buff *skb, const bool ls) { u32 desc_idx = ei_local->txd_last_idx; while (frag_size > 0) { const u32 part_addr = (u32)frag_addr; const u32 part_size = min_t(u32, frag_size, TXD_MAX_SEG_SIZE); struct PDMA_txdesc *txd; frag_size -= part_size; frag_addr += part_size; txd = &ei_local->txd_ring[desc_idx]; if (*desc_odd == 0) { *txd_info2 = TX2_DMA_SDL0(part_size); ACCESS_ONCE(txd->txd_info1) = part_addr; ACCESS_ONCE(txd->txd_info4) = txd_info4; if (ls && frag_size == 0) { *txd_info2 |= TX2_DMA_LS0; ACCESS_ONCE(txd->txd_info3) = 0; ACCESS_ONCE(txd->txd_info2) = *txd_info2; /* store skb with LS0 bit */ ei_local->txd_buff[desc_idx] = skb; desc_idx = (desc_idx + 1) % NUM_TX_DESC; } else *desc_odd = 1; } else { *txd_info2 |= TX2_DMA_SDL1(part_size); if (ls && frag_size == 0) { *txd_info2 |= TX2_DMA_LS1; /* store skb with LS1 bit */ ei_local->txd_buff[desc_idx] = skb; } else ei_local->txd_buff[desc_idx] = (struct sk_buff *)0xFFFFFFFF; //MAGIC ID ACCESS_ONCE(txd->txd_info3) = part_addr; ACCESS_ONCE(txd->txd_info2) = *txd_info2; desc_idx = (desc_idx + 1) % NUM_TX_DESC; *desc_odd = 0; } } ei_local->txd_last_idx = desc_idx; }
static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, struct iwl_rx_mem_buffer *rxb) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; unsigned long flags; bool page_stolen = false; int max_len = PAGE_SIZE << trans_pcie->rx_page_order; u32 offset = 0; if (WARN_ON(!rxb)) return; dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE); while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) { struct iwl_rx_packet *pkt; struct iwl_device_cmd *cmd; u16 sequence; bool reclaim; int index, cmd_index, err, len; struct iwl_rx_cmd_buffer rxcb = { ._offset = offset, ._rx_page_order = trans_pcie->rx_page_order, ._page = rxb->page, ._page_stolen = false, .truesize = max_len, }; pkt = rxb_addr(&rxcb); if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) break; IWL_DEBUG_RX(trans, "cmd at offset %d: %s (0x%.2x)\n", rxcb._offset, get_cmd_string(trans_pcie, pkt->hdr.cmd), pkt->hdr.cmd); len = le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_FRAME_SIZE_MSK; len += sizeof(u32); /* account for status word */ trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len); /* Reclaim a command buffer only if this packet is a response * to a (driver-originated) command. * If the packet (e.g. Rx frame) originated from uCode, * there is no command buffer to reclaim. * Ucode should set SEQ_RX_FRAME bit if ucode-originated, * but apparently a few don't get set; catch them here. */ reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME); if (reclaim) { int i; for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) { if (trans_pcie->no_reclaim_cmds[i] == pkt->hdr.cmd) { reclaim = false; break; } } } sequence = le16_to_cpu(pkt->hdr.sequence); index = SEQ_TO_INDEX(sequence); cmd_index = get_cmd_index(&txq->q, index); if (reclaim) cmd = txq->entries[cmd_index].cmd; else cmd = NULL; err = iwl_op_mode_rx(trans->op_mode, &rxcb, cmd); if (reclaim) { kfree(txq->entries[cmd_index].free_buf); txq->entries[cmd_index].free_buf = NULL; } /* * After here, we should always check rxcb._page_stolen, * if it is true then one of the handlers took the page. */ if (reclaim) { /* Invoke any callbacks, transfer the buffer to caller, * and fire off the (possibly) blocking * iwl_trans_send_cmd() * as we reclaim the driver command queue */ if (!rxcb._page_stolen) iwl_pcie_hcmd_complete(trans, &rxcb, err); else IWL_WARN(trans, "Claim null rxb?\n"); } page_stolen |= rxcb._page_stolen; offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); } /* page was stolen from us -- free our reference */ if (page_stolen) { __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; } /* Reuse the page if possible. For notification packets and * SKBs that fail to Rx correctly, add them back into the * rx_free list for reuse later. */ spin_lock_irqsave(&rxq->lock, flags); if (rxb->page != NULL) { rxb->page_dma = dma_map_page(trans->dev, rxb->page, 0, PAGE_SIZE << trans_pcie->rx_page_order, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { /* * free the page(s) as well to not break * the invariant that the items on the used * list have no page(s) */ __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; list_add_tail(&rxb->list, &rxq->rx_used); } else { list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; } } else list_add_tail(&rxb->list, &rxq->rx_used); spin_unlock_irqrestore(&rxq->lock, flags); } /* * iwl_pcie_rx_handle - Main entry function for receiving responses from fw */ static void iwl_pcie_rx_handle(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; u32 r, i; u8 fill_rx = 0; u32 count = 8; int total_empty; /* uCode's read index (stored in shared DRAM) indicates the last Rx * buffer that the driver may process (last buffer filled by ucode). */ r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF; i = rxq->read; /* Rx interrupt, but nothing sent from uCode */ if (i == r) IWL_DEBUG_RX(trans, "HW = SW = %d\n", r); /* calculate total frames need to be restock after handling RX */ total_empty = r - rxq->write_actual; if (total_empty < 0) total_empty += RX_QUEUE_SIZE; if (total_empty > (RX_QUEUE_SIZE / 2)) fill_rx = 1; while (i != r) { struct iwl_rx_mem_buffer *rxb; rxb = rxq->queue[i]; rxq->queue[i] = NULL; IWL_DEBUG_RX(trans, "rxbuf: HW = %d, SW = %d (%p)\n", r, i, rxb); iwl_pcie_rx_handle_rb(trans, rxb); i = (i + 1) & RX_QUEUE_MASK; /* If there are a lot of unused frames, * restock the Rx queue so ucode wont assert. */ if (fill_rx) { count++; if (count >= 8) { rxq->read = i; iwl_pcie_rx_replenish_now(trans); count = 0; } } } /* Backtrack one entry */ rxq->read = i; if (fill_rx) iwl_pcie_rx_replenish_now(trans); else iwl_pcie_rxq_restock(trans); } /* * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card */ static void iwl_pcie_irq_handle_error(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); /* W/A for WiFi/WiMAX coex and WiMAX own the RF */ if (trans->cfg->internal_wimax_coex && (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) & APMS_CLK_VAL_MRB_FUNC_MODE) || (iwl_read_prph(trans, APMG_PS_CTRL_REG) & APMG_PS_CTRL_VAL_RESET_REQ))) { clear_bit(STATUS_HCMD_ACTIVE, &trans_pcie->status); iwl_op_mode_wimax_active(trans->op_mode); wake_up(&trans_pcie->wait_command_queue); return; } iwl_pcie_dump_csr(trans); iwl_pcie_dump_fh(trans, NULL); set_bit(STATUS_FW_ERROR, &trans_pcie->status); clear_bit(STATUS_HCMD_ACTIVE, &trans_pcie->status); wake_up(&trans_pcie->wait_command_queue); local_bh_disable(); iwl_op_mode_nic_error(trans->op_mode); local_bh_enable(); } irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) { struct iwl_trans *trans = dev_id; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct isr_statistics *isr_stats = &trans_pcie->isr_stats; u32 inta = 0; u32 handled = 0; unsigned long flags; u32 i; lock_map_acquire(&trans->sync_cmd_lockdep_map); spin_lock_irqsave(&trans_pcie->irq_lock, flags); /* Ack/clear/reset pending uCode interrupts. * Note: Some bits in CSR_INT are "OR" of bits in CSR_FH_INT_STATUS, */ /* There is a hardware bug in the interrupt mask function that some * interrupts (i.e. CSR_INT_BIT_SCD) can still be generated even if * they are disabled in the CSR_INT_MASK register. Furthermore the * ICT interrupt handling mechanism has another bug that might cause * these unmasked interrupts fail to be detected. We workaround the * hardware bugs here by ACKing all the possible interrupts so that * interrupt coalescing can still be achieved. */ iwl_write32(trans, CSR_INT, trans_pcie->inta | ~trans_pcie->inta_mask); inta = trans_pcie->inta; if (iwl_have_debug_level(IWL_DL_ISR)) IWL_DEBUG_ISR(trans, "inta 0x%08x, enabled 0x%08x\n", inta, iwl_read32(trans, CSR_INT_MASK)); /* saved interrupt in inta variable now we can reset trans_pcie->inta */ trans_pcie->inta = 0; spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); /* Now service all interrupt bits discovered above. */ if (inta & CSR_INT_BIT_HW_ERR) { IWL_ERR(trans, "Hardware error detected. Restarting.\n"); /* Tell the device to stop sending interrupts */ iwl_disable_interrupts(trans); isr_stats->hw++; iwl_pcie_irq_handle_error(trans); handled |= CSR_INT_BIT_HW_ERR; goto out; } if (iwl_have_debug_level(IWL_DL_ISR)) { /* NIC fires this, but we don't use it, redundant with WAKEUP */ if (inta & CSR_INT_BIT_SCD) { IWL_DEBUG_ISR(trans, "Scheduler finished to transmit the frame/frames.\n"); isr_stats->sch++; } /* Alive notification via Rx interrupt will do the real work */ if (inta & CSR_INT_BIT_ALIVE) { IWL_DEBUG_ISR(trans, "Alive interrupt\n"); isr_stats->alive++; } } /* Safely ignore these bits for debug checks below */ inta &= ~(CSR_INT_BIT_SCD | CSR_INT_BIT_ALIVE); /* HW RF KILL switch toggled */ if (inta & CSR_INT_BIT_RF_KILL) { bool hw_rfkill; hw_rfkill = iwl_is_rfkill_set(trans); IWL_WARN(trans, "RF_KILL bit toggled to %s.\n", hw_rfkill ? "disable radio" : "enable radio"); isr_stats->rfkill++; iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill); if (hw_rfkill) { /* * Clear the interrupt in APMG if the NIC is going down. * Note that when the NIC exits RFkill (else branch), we * can't access prph and the NIC will be reset in * start_hw anyway. */ iwl_write_prph(trans, APMG_RTC_INT_STT_REG, APMG_RTC_INT_STT_RFKILL); set_bit(STATUS_RFKILL, &trans_pcie->status); if (test_and_clear_bit(STATUS_HCMD_ACTIVE, &trans_pcie->status)) IWL_DEBUG_RF_KILL(trans, "Rfkill while SYNC HCMD in flight\n"); wake_up(&trans_pcie->wait_command_queue); } else { clear_bit(STATUS_RFKILL, &trans_pcie->status); } handled |= CSR_INT_BIT_RF_KILL; } /* Chip got too hot and stopped itself */ if (inta & CSR_INT_BIT_CT_KILL) { IWL_ERR(trans, "Microcode CT kill error detected.\n"); isr_stats->ctkill++; handled |= CSR_INT_BIT_CT_KILL; } /* Error detected by uCode */ if (inta & CSR_INT_BIT_SW_ERR) { IWL_ERR(trans, "Microcode SW error detected. " " Restarting 0x%X.\n", inta); isr_stats->sw++; iwl_pcie_irq_handle_error(trans); handled |= CSR_INT_BIT_SW_ERR; } /* uCode wakes up after power-down sleep */ if (inta & CSR_INT_BIT_WAKEUP) { IWL_DEBUG_ISR(trans, "Wakeup interrupt\n"); iwl_pcie_rxq_inc_wr_ptr(trans, &trans_pcie->rxq); for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) iwl_pcie_txq_inc_wr_ptr(trans, &trans_pcie->txq[i]); isr_stats->wakeup++; handled |= CSR_INT_BIT_WAKEUP; } /* All uCode command responses, including Tx command responses, * Rx "responses" (frame-received notification), and other * notifications from uCode come through here*/ if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX | CSR_INT_BIT_RX_PERIODIC)) { IWL_DEBUG_ISR(trans, "Rx interrupt\n"); if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) { handled |= (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX); iwl_write32(trans, CSR_FH_INT_STATUS, CSR_FH_INT_RX_MASK); } if (inta & CSR_INT_BIT_RX_PERIODIC) { handled |= CSR_INT_BIT_RX_PERIODIC; iwl_write32(trans, CSR_INT, CSR_INT_BIT_RX_PERIODIC); } /* Sending RX interrupt require many steps to be done in the * the device: * 1- write interrupt to current index in ICT table. * 2- dma RX frame. * 3- update RX shared data to indicate last write index. * 4- send interrupt. * This could lead to RX race, driver could receive RX interrupt * but the shared data changes does not reflect this; * periodic interrupt will detect any dangling Rx activity. */ /* Disable periodic interrupt; we use it as just a one-shot. */ iwl_write8(trans, CSR_INT_PERIODIC_REG, CSR_INT_PERIODIC_DIS); iwl_pcie_rx_handle(trans); /* * Enable periodic interrupt in 8 msec only if we received * real RX interrupt (instead of just periodic int), to catch * any dangling Rx interrupt. If it was just the periodic * interrupt, there was no dangling Rx activity, and no need * to extend the periodic interrupt; one-shot is enough. */ if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) iwl_write8(trans, CSR_INT_PERIODIC_REG, CSR_INT_PERIODIC_ENA); isr_stats->rx++; } /* This "Tx" DMA channel is used only for loading uCode */ if (inta & CSR_INT_BIT_FH_TX) { iwl_write32(trans, CSR_FH_INT_STATUS, CSR_FH_INT_TX_MASK); IWL_DEBUG_ISR(trans, "uCode load interrupt\n"); isr_stats->tx++; handled |= CSR_INT_BIT_FH_TX; /* Wake up uCode load routine, now that load is complete */ trans_pcie->ucode_write_complete = true; wake_up(&trans_pcie->ucode_write_waitq); } if (inta & ~handled) { IWL_ERR(trans, "Unhandled INTA bits 0x%08x\n", inta & ~handled); isr_stats->unhandled++; } if (inta & ~(trans_pcie->inta_mask)) { IWL_WARN(trans, "Disabled INTA bits 0x%08x were pending\n", inta & ~trans_pcie->inta_mask); } /* Re-enable all interrupts */ /* only Re-enable if disabled by irq */ if (test_bit(STATUS_INT_ENABLED, &trans_pcie->status)) iwl_enable_interrupts(trans); /* Re-enable RF_KILL if it occurred */ else if (handled & CSR_INT_BIT_RF_KILL) iwl_enable_rfkill_int(trans); out: lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_HANDLED; } /****************************************************************************** * * ICT functions * ******************************************************************************/ /* a device (PCI-E) page is 4096 bytes long */ #define ICT_SHIFT 12 #define ICT_SIZE (1 << ICT_SHIFT) #define ICT_COUNT (ICT_SIZE / sizeof(u32)) /* Free dram table */ void iwl_pcie_free_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); if (trans_pcie->ict_tbl) { dma_free_coherent(trans->dev, ICT_SIZE, trans_pcie->ict_tbl, trans_pcie->ict_tbl_dma); trans_pcie->ict_tbl = NULL; trans_pcie->ict_tbl_dma = 0; } } /* * allocate dram shared table, it is an aligned memory * block of ICT_SIZE. * also reset all data related to ICT table interrupt. */ int iwl_pcie_alloc_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); trans_pcie->ict_tbl = dma_alloc_coherent(trans->dev, ICT_SIZE, &trans_pcie->ict_tbl_dma, GFP_KERNEL); if (!trans_pcie->ict_tbl) return -ENOMEM; /* just an API sanity check ... it is guaranteed to be aligned */ if (WARN_ON(trans_pcie->ict_tbl_dma & (ICT_SIZE - 1))) { iwl_pcie_free_ict(trans); return -EINVAL; } IWL_DEBUG_ISR(trans, "ict dma addr %Lx\n", (unsigned long long)trans_pcie->ict_tbl_dma); IWL_DEBUG_ISR(trans, "ict vir addr %p\n", trans_pcie->ict_tbl); /* reset table and index to all 0 */ memset(trans_pcie->ict_tbl, 0, ICT_SIZE); trans_pcie->ict_index = 0; /* add periodic RX interrupt */ trans_pcie->inta_mask |= CSR_INT_BIT_RX_PERIODIC; return 0; } /* Device is going up inform it about using ICT interrupt table, * also we need to tell the driver to start using ICT interrupt. */ void iwl_pcie_reset_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 val; unsigned long flags; if (!trans_pcie->ict_tbl) return; spin_lock_irqsave(&trans_pcie->irq_lock, flags); iwl_disable_interrupts(trans); memset(trans_pcie->ict_tbl, 0, ICT_SIZE); val = trans_pcie->ict_tbl_dma >> ICT_SHIFT; val |= CSR_DRAM_INT_TBL_ENABLE; val |= CSR_DRAM_INIT_TBL_WRAP_CHECK; IWL_DEBUG_ISR(trans, "CSR_DRAM_INT_TBL_REG =0x%x\n", val); iwl_write32(trans, CSR_DRAM_INT_TBL_REG, val); trans_pcie->use_ict = true; trans_pcie->ict_index = 0; iwl_write32(trans, CSR_INT, trans_pcie->inta_mask); iwl_enable_interrupts(trans); spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); } /* Device is going down disable ict interrupt usage */ void iwl_pcie_disable_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); unsigned long flags; spin_lock_irqsave(&trans_pcie->irq_lock, flags); trans_pcie->use_ict = false; spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); } /* legacy (non-ICT) ISR. Assumes that trans_pcie->irq_lock is held */ static irqreturn_t iwl_pcie_isr(int irq, void *data) { struct iwl_trans *trans = data; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 inta, inta_mask; lockdep_assert_held(&trans_pcie->irq_lock); trace_iwlwifi_dev_irq(trans->dev); /* Disable (but don't clear!) interrupts here to avoid * back-to-back ISRs and sporadic interrupts from our NIC. * If we have something to service, the irq thread will re-enable ints. * If we *don't* have something, we'll re-enable before leaving here. */ inta_mask = iwl_read32(trans, CSR_INT_MASK); iwl_write32(trans, CSR_INT_MASK, 0x00000000); /* Discover which interrupts are active/pending */ inta = iwl_read32(trans, CSR_INT); if (inta & (~inta_mask)) { IWL_DEBUG_ISR(trans, "We got a masked interrupt (0x%08x)...Ack and ignore\n", inta & (~inta_mask)); iwl_write32(trans, CSR_INT, inta & (~inta_mask)); inta &= inta_mask; } /* Ignore interrupt if there's nothing in NIC to service. * This may be due to IRQ shared with another device, * or due to sporadic interrupts thrown from our NIC. */ if (!inta) { IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n"); goto none; } if ((inta == 0xFFFFFFFF) || ((inta & 0xFFFFFFF0) == 0xa5a5a5a0)) { /* Hardware disappeared. It might have already raised * an interrupt */ IWL_WARN(trans, "HARDWARE GONE?? INTA == 0x%08x\n", inta); return IRQ_HANDLED; } if (iwl_have_debug_level(IWL_DL_ISR)) IWL_DEBUG_ISR(trans, "ISR inta 0x%08x, enabled 0x%08x, fh 0x%08x\n", inta, inta_mask, iwl_read32(trans, CSR_FH_INT_STATUS)); trans_pcie->inta |= inta; /* the thread will service interrupts and re-enable them */ if (likely(inta)) return IRQ_WAKE_THREAD; else if (test_bit(STATUS_INT_ENABLED, &trans_pcie->status) && !trans_pcie->inta) iwl_enable_interrupts(trans); return IRQ_HANDLED; none: /* re-enable interrupts here since we don't have anything to service. */ /* only Re-enable if disabled by irq and no schedules tasklet. */ if (test_bit(STATUS_INT_ENABLED, &trans_pcie->status) && !trans_pcie->inta) iwl_enable_interrupts(trans); return IRQ_NONE; } /* interrupt handler using ict table, with this interrupt driver will * stop using INTA register to get device's interrupt, reading this register * is expensive, device will write interrupts in ICT dram table, increment * index then will fire interrupt to driver, driver will OR all ICT table * entries from current index up to table entry with 0 value. the result is * the interrupt we need to service, driver will set the entries back to 0 and * set index. */ irqreturn_t iwl_pcie_isr_ict(int irq, void *data) { struct iwl_trans *trans = data; struct iwl_trans_pcie *trans_pcie; u32 inta; u32 val = 0; u32 read; unsigned long flags; if (!trans) return IRQ_NONE; trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); spin_lock_irqsave(&trans_pcie->irq_lock, flags); /* dram interrupt table not set yet, * use legacy interrupt. */ if (unlikely(!trans_pcie->use_ict)) { irqreturn_t ret = iwl_pcie_isr(irq, data); spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); return ret; } trace_iwlwifi_dev_irq(trans->dev); /* Disable (but don't clear!) interrupts here to avoid * back-to-back ISRs and sporadic interrupts from our NIC. * If we have something to service, the tasklet will re-enable ints. * If we *don't* have something, we'll re-enable before leaving here. */ iwl_write32(trans, CSR_INT_MASK, 0x00000000); /* Ignore interrupt if there's nothing in NIC to service. * This may be due to IRQ shared with another device, * or due to sporadic interrupts thrown from our NIC. */ read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]); trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read); if (!read) { IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n"); goto none; } /* * Collect all entries up to the first 0, starting from ict_index; * note we already read at ict_index. */ do { val |= read; IWL_DEBUG_ISR(trans, "ICT index %d value 0x%08X\n", trans_pcie->ict_index, read); trans_pcie->ict_tbl[trans_pcie->ict_index] = 0; trans_pcie->ict_index = iwl_queue_inc_wrap(trans_pcie->ict_index, ICT_COUNT); read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]); trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read); } while (read); /* We should not get this value, just ignore it. */ if (val == 0xffffffff) val = 0; /* * this is a w/a for a h/w bug. the h/w bug may cause the Rx bit * (bit 15 before shifting it to 31) to clear when using interrupt * coalescing. fortunately, bits 18 and 19 stay set when this happens * so we use them to decide on the real state of the Rx bit. * In order words, bit 15 is set if bit 18 or bit 19 are set. */ if (val & 0xC0000) val |= 0x8000; inta = (0xff & val) | ((0xff00 & val) << 16); IWL_DEBUG_ISR(trans, "ISR inta 0x%08x, enabled(sw) 0x%08x ict 0x%08x\n", inta, trans_pcie->inta_mask, val); if (iwl_have_debug_level(IWL_DL_ISR)) IWL_DEBUG_ISR(trans, "enabled(hw) 0x%08x\n", iwl_read32(trans, CSR_INT_MASK)); inta &= trans_pcie->inta_mask; trans_pcie->inta |= inta; /* iwl_pcie_tasklet() will service interrupts and re-enable them */ if (likely(inta)) { spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); return IRQ_WAKE_THREAD; } else if (test_bit(STATUS_INT_ENABLED, &trans_pcie->status) && !trans_pcie->inta) { /* Allow interrupt if was disabled by this handler and * no tasklet was schedules, We should not enable interrupt, * tasklet will enable it. */ iwl_enable_interrupts(trans); } spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); return IRQ_HANDLED; none: /* re-enable interrupts here since we don't have anything to service. * only Re-enable if disabled by irq. */ if (test_bit(STATUS_INT_ENABLED, &trans_pcie->status) && !trans_pcie->inta) iwl_enable_interrupts(trans); spin_unlock_irqrestore(&trans_pcie->irq_lock, flags); return IRQ_NONE; }
static void ar9002_set_txdesc(struct ath_hw *ah, void *ds, struct ath_tx_info *i) { struct ar5416_desc *ads = AR5416DESC(ds); u32 ctl1, ctl6; ads->ds_txstatus0 = ads->ds_txstatus1 = 0; ads->ds_txstatus2 = ads->ds_txstatus3 = 0; ads->ds_txstatus4 = ads->ds_txstatus5 = 0; ads->ds_txstatus6 = ads->ds_txstatus7 = 0; ads->ds_txstatus8 = ads->ds_txstatus9 = 0; ACCESS_ONCE(ads->ds_link) = i->link; ACCESS_ONCE(ads->ds_data) = i->buf_addr[0]; ctl1 = i->buf_len[0] | (i->is_last ? 0 : AR_TxMore); ctl6 = SM(i->keytype, AR_EncrType); if (AR_SREV_9285(ah)) { ads->ds_ctl8 = 0; ads->ds_ctl9 = 0; ads->ds_ctl10 = 0; ads->ds_ctl11 = 0; } if ((i->is_first || i->is_last) && i->aggr != AGGR_BUF_MIDDLE && i->aggr != AGGR_BUF_LAST) { ACCESS_ONCE(ads->ds_ctl2) = set11nTries(i->rates, 0) | set11nTries(i->rates, 1) | set11nTries(i->rates, 2) | set11nTries(i->rates, 3) | (i->dur_update ? AR_DurUpdateEna : 0) | SM(0, AR_BurstDur); ACCESS_ONCE(ads->ds_ctl3) = set11nRate(i->rates, 0) | set11nRate(i->rates, 1) | set11nRate(i->rates, 2) | set11nRate(i->rates, 3); } else { ACCESS_ONCE(ads->ds_ctl2) = 0; ACCESS_ONCE(ads->ds_ctl3) = 0; } if (!i->is_first) { ACCESS_ONCE(ads->ds_ctl0) = 0; ACCESS_ONCE(ads->ds_ctl1) = ctl1; ACCESS_ONCE(ads->ds_ctl6) = ctl6; return; } ctl1 |= (i->keyix != ATH9K_TXKEYIX_INVALID ? SM(i->keyix, AR_DestIdx) : 0) | SM(i->type, AR_FrameType) | (i->flags & ATH9K_TXDESC_NOACK ? AR_NoAck : 0) | (i->flags & ATH9K_TXDESC_EXT_ONLY ? AR_ExtOnly : 0) | (i->flags & ATH9K_TXDESC_EXT_AND_CTL ? AR_ExtAndCtl : 0); switch (i->aggr) { case AGGR_BUF_FIRST: ctl6 |= SM(i->aggr_len, AR_AggrLen); /* fall through */ case AGGR_BUF_MIDDLE: ctl1 |= AR_IsAggr | AR_MoreAggr; ctl6 |= SM(i->ndelim, AR_PadDelim); break; case AGGR_BUF_LAST: ctl1 |= AR_IsAggr; break; case AGGR_BUF_NONE: break; } ACCESS_ONCE(ads->ds_ctl0) = (i->pkt_len & AR_FrameLen) | (i->flags & ATH9K_TXDESC_VMF ? AR_VirtMoreFrag : 0) | SM(i->txpower, AR_XmitPower) | (i->flags & ATH9K_TXDESC_VEOL ? AR_VEOL : 0) | (i->flags & ATH9K_TXDESC_INTREQ ? AR_TxIntrReq : 0) | (i->keyix != ATH9K_TXKEYIX_INVALID ? AR_DestIdxValid : 0) | (i->flags & ATH9K_TXDESC_CLRDMASK ? AR_ClrDestMask : 0) | (i->flags & ATH9K_TXDESC_RTSENA ? AR_RTSEnable : (i->flags & ATH9K_TXDESC_CTSENA ? AR_CTSEnable : 0)); ACCESS_ONCE(ads->ds_ctl1) = ctl1; ACCESS_ONCE(ads->ds_ctl6) = ctl6; if (i->aggr == AGGR_BUF_MIDDLE || i->aggr == AGGR_BUF_LAST) return; ACCESS_ONCE(ads->ds_ctl4) = set11nPktDurRTSCTS(i->rates, 0) | set11nPktDurRTSCTS(i->rates, 1); ACCESS_ONCE(ads->ds_ctl5) = set11nPktDurRTSCTS(i->rates, 2) | set11nPktDurRTSCTS(i->rates, 3); ACCESS_ONCE(ads->ds_ctl7) = set11nRateFlags(i->rates, 0) | set11nRateFlags(i->rates, 1) | set11nRateFlags(i->rates, 2) | set11nRateFlags(i->rates, 3) | SM(i->rtscts_rate, AR_RTSCTSRate); }
static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, struct iwl_rx_mem_buffer *rxb, bool emergency) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; bool page_stolen = false; int max_len = PAGE_SIZE << trans_pcie->rx_page_order; u32 offset = 0; if (WARN_ON(!rxb)) return; dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE); while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) { struct iwl_rx_packet *pkt; u16 sequence; bool reclaim; int index, cmd_index, len; struct iwl_rx_cmd_buffer rxcb = { ._offset = offset, ._rx_page_order = trans_pcie->rx_page_order, ._page = rxb->page, ._page_stolen = false, .truesize = max_len, }; pkt = rxb_addr(&rxcb); if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) break; IWL_DEBUG_RX(trans, "cmd at offset %d: %s (0x%.2x, seq 0x%x)\n", rxcb._offset, iwl_get_cmd_string(trans, iwl_cmd_id(pkt->hdr.cmd, pkt->hdr.group_id, 0)), pkt->hdr.cmd, le16_to_cpu(pkt->hdr.sequence)); len = iwl_rx_packet_len(pkt); len += sizeof(u32); /* account for status word */ trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len); /* Reclaim a command buffer only if this packet is a response * to a (driver-originated) command. * If the packet (e.g. Rx frame) originated from uCode, * there is no command buffer to reclaim. * Ucode should set SEQ_RX_FRAME bit if ucode-originated, * but apparently a few don't get set; catch them here. */ reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME); if (reclaim) { int i; for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) { if (trans_pcie->no_reclaim_cmds[i] == pkt->hdr.cmd) { reclaim = false; break; } } } sequence = le16_to_cpu(pkt->hdr.sequence); index = SEQ_TO_INDEX(sequence); cmd_index = get_cmd_index(&txq->q, index); iwl_op_mode_rx(trans->op_mode, &trans_pcie->napi, &rxcb); if (reclaim) { kzfree(txq->entries[cmd_index].free_buf); txq->entries[cmd_index].free_buf = NULL; } /* * After here, we should always check rxcb._page_stolen, * if it is true then one of the handlers took the page. */ if (reclaim) { /* Invoke any callbacks, transfer the buffer to caller, * and fire off the (possibly) blocking * iwl_trans_send_cmd() * as we reclaim the driver command queue */ if (!rxcb._page_stolen) iwl_pcie_hcmd_complete(trans, &rxcb); else IWL_WARN(trans, "Claim null rxb?\n"); } page_stolen |= rxcb._page_stolen; offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); } /* page was stolen from us -- free our reference */ if (page_stolen) { __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; } /* Reuse the page if possible. For notification packets and * SKBs that fail to Rx correctly, add them back into the * rx_free list for reuse later. */ if (rxb->page != NULL) { rxb->page_dma = dma_map_page(trans->dev, rxb->page, 0, PAGE_SIZE << trans_pcie->rx_page_order, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { /* * free the page(s) as well to not break * the invariant that the items on the used * list have no page(s) */ __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency); } else { list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; } } else iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency); } /* * iwl_pcie_rx_handle - Main entry function for receiving responses from fw */ static void iwl_pcie_rx_handle(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq = &trans_pcie->rxq; u32 r, i, j, count = 0; bool emergency = false; restart: spin_lock(&rxq->lock); /* uCode's read index (stored in shared DRAM) indicates the last Rx * buffer that the driver may process (last buffer filled by ucode). */ r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF; i = rxq->read; /* Rx interrupt, but nothing sent from uCode */ if (i == r) IWL_DEBUG_RX(trans, "HW = SW = %d\n", r); while (i != r) { struct iwl_rx_mem_buffer *rxb; if (unlikely(rxq->used_count == RX_QUEUE_SIZE / 2)) emergency = true; rxb = rxq->queue[i]; rxq->queue[i] = NULL; IWL_DEBUG_RX(trans, "rxbuf: HW = %d, SW = %d\n", r, i); iwl_pcie_rx_handle_rb(trans, rxb, emergency); i = (i + 1) & RX_QUEUE_MASK; /* If we have RX_CLAIM_REQ_ALLOC released rx buffers - * try to claim the pre-allocated buffers from the allocator */ if (rxq->used_count >= RX_CLAIM_REQ_ALLOC) { struct iwl_rb_allocator *rba = &trans_pcie->rba; struct iwl_rx_mem_buffer *out[RX_CLAIM_REQ_ALLOC]; if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 && !emergency) { /* Add the remaining 6 empty RBDs * for allocator use */ spin_lock(&rba->lock); list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty); spin_unlock(&rba->lock); } /* If not ready - continue, will try to reclaim later. * No need to reschedule work - allocator exits only on * success */ if (!iwl_pcie_rx_allocator_get(trans, out)) { /* If success - then RX_CLAIM_REQ_ALLOC * buffers were retrieved and should be added * to free list */ rxq->used_count -= RX_CLAIM_REQ_ALLOC; for (j = 0; j < RX_CLAIM_REQ_ALLOC; j++) { list_add_tail(&out[j]->list, &rxq->rx_free); rxq->free_count++; } } } if (emergency) { count++; if (count == 8) { count = 0; if (rxq->used_count < RX_QUEUE_SIZE / 3) emergency = false; spin_unlock(&rxq->lock); iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC); spin_lock(&rxq->lock); } } /* handle restock for three cases, can be all of them at once: * - we just pulled buffers from the allocator * - we have 8+ unstolen pages accumulated * - we are in emergency and allocated buffers */ if (rxq->free_count >= RX_CLAIM_REQ_ALLOC) { rxq->read = i; spin_unlock(&rxq->lock); iwl_pcie_rxq_restock(trans); goto restart; } } /* Backtrack one entry */ rxq->read = i; spin_unlock(&rxq->lock); /* * handle a case where in emergency there are some unallocated RBDs. * those RBDs are in the used list, but are not tracked by the queue's * used_count which counts allocator owned RBDs. * unallocated emergency RBDs must be allocated on exit, otherwise * when called again the function may not be in emergency mode and * they will be handed to the allocator with no tracking in the RBD * allocator counters, which will lead to them never being claimed back * by the queue. * by allocating them here, they are now in the queue free list, and * will be restocked by the next call of iwl_pcie_rxq_restock. */ if (unlikely(emergency && count)) iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC); if (trans_pcie->napi.poll) napi_gro_flush(&trans_pcie->napi, false); } /* * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card */ static void iwl_pcie_irq_handle_error(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int i; /* W/A for WiFi/WiMAX coex and WiMAX own the RF */ if (trans->cfg->internal_wimax_coex && !trans->cfg->apmg_not_supported && (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) & APMS_CLK_VAL_MRB_FUNC_MODE) || (iwl_read_prph(trans, APMG_PS_CTRL_REG) & APMG_PS_CTRL_VAL_RESET_REQ))) { clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); iwl_op_mode_wimax_active(trans->op_mode); wake_up(&trans_pcie->wait_command_queue); return; } iwl_pcie_dump_csr(trans); iwl_dump_fh(trans, NULL); local_bh_disable(); /* The STATUS_FW_ERROR bit is set in this function. This must happen * before we wake up the command caller, to ensure a proper cleanup. */ iwl_trans_fw_error(trans); local_bh_enable(); for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) del_timer(&trans_pcie->txq[i].stuck_timer); clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); wake_up(&trans_pcie->wait_command_queue); } static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans) { u32 inta; lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock); trace_iwlwifi_dev_irq(trans->dev); /* Discover which interrupts are active/pending */ inta = iwl_read32(trans, CSR_INT); /* the thread will service interrupts and re-enable them */ return inta; }
/** * ixgbe_ptp_start_cyclecounter - create the cycle counter from hw * @adapter: pointer to the adapter structure * * This function should be called to set the proper values for the TIMINCA * register and tell the cyclecounter structure what the tick rate of SYSTIME * is. It does not directly modify SYSTIME registers or the timecounter * structure. It should be called whenever a new TIMINCA value is necessary, * such as during initialization or when the link speed changes. */ void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 incval = 0; u32 shift = 0; unsigned long flags; /** * Scale the NIC cycle counter by a large factor so that * relatively small corrections to the frequency can be added * or subtracted. The drawbacks of a large factor include * (a) the clock register overflows more quickly, (b) the cycle * counter structure must be able to convert the systime value * to nanoseconds using only a multiplier and a right-shift, * and (c) the value must fit within the timinca register space * => math based on internal DMA clock rate and available bits * * Note that when there is no link, internal DMA clock is same as when * link speed is 10Gb. Set the registers correctly even when link is * down to preserve the clock setting */ switch (adapter->link_speed) { case IXGBE_LINK_SPEED_100_FULL: incval = IXGBE_INCVAL_100; shift = IXGBE_INCVAL_SHIFT_100; break; case IXGBE_LINK_SPEED_1GB_FULL: incval = IXGBE_INCVAL_1GB; shift = IXGBE_INCVAL_SHIFT_1GB; break; case IXGBE_LINK_SPEED_10GB_FULL: default: incval = IXGBE_INCVAL_10GB; shift = IXGBE_INCVAL_SHIFT_10GB; break; } /** * Modify the calculated values to fit within the correct * number of bits specified by the hardware. The 82599 doesn't * have the same space as the X540, so bitshift the calculated * values to fit. */ switch (hw->mac.type) { case ixgbe_mac_X540: IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval); break; case ixgbe_mac_82599EB: incval >>= IXGBE_INCVAL_SHIFT_82599; shift -= IXGBE_INCVAL_SHIFT_82599; IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, (1 << IXGBE_INCPER_SHIFT_82599) | incval); break; default: /* other devices aren't supported */ return; } /* update the base incval used to calculate frequency adjustment */ ACCESS_ONCE(adapter->base_incval) = incval; smp_mb(); /* need lock to prevent incorrect read while modifying cyclecounter */ spin_lock_irqsave(&adapter->tmreg_lock, flags); memset(&adapter->hw_cc, 0, sizeof(adapter->hw_cc)); adapter->hw_cc.read = ixgbe_ptp_read_82599; adapter->hw_cc.mask = CLOCKSOURCE_MASK(64); adapter->hw_cc.shift = shift; adapter->hw_cc.mult = 1; spin_unlock_irqrestore(&adapter->tmreg_lock, flags); }
unsigned long read_count(void) { return ACCESS_ONCE(global_count); }
/* Records completed count and recalculates the queue limit */ void dql_completed(struct dql *dql, unsigned int count) { unsigned int inprogress, prev_inprogress, limit; //Update Patch from Google //https://android.googlesource.com/kernel/common/+/1414a53d956340ca8b1b27e05ab94ba63e82ed97%5E..1414a53d956340ca8b1b27e05ab94ba63e82ed97/#F0 //unsigned int ovlimit, all_prev_completed, completed; //Update Patch from Google //https://android.googlesource.com/kernel/common/+/4f4bdaeb40df95499c1ee7ea3fbca9d76174a59e%5E..4f4bdaeb40df95499c1ee7ea3fbca9d76174a59e/#F0 unsigned int ovlimit, completed, num_queued; bool all_prev_completed; num_queued = ACCESS_ONCE(dql->num_queued); /* Can't complete more than what's in queue */ BUG_ON(count > num_queued - dql->num_completed); completed = dql->num_completed + count; limit = dql->limit; ovlimit = POSDIFF(num_queued - dql->num_completed, limit); inprogress = num_queued - completed; prev_inprogress = dql->prev_num_queued - dql->num_completed; all_prev_completed = AFTER_EQ(completed, dql->prev_num_queued); if ((ovlimit && !inprogress) || (dql->prev_ovlimit && all_prev_completed)) { /* * Queue considered starved if: * - The queue was over-limit in the last interval, * and there is no more data in the queue. * OR * - The queue was over-limit in the previous interval and * when enqueuing it was possible that all queued data * had been consumed. This covers the case when queue * may have becomes starved between completion processing * running and next time enqueue was scheduled. * * When queue is starved increase the limit by the amount * of bytes both sent and completed in the last interval, * plus any previous over-limit. */ limit += POSDIFF(completed, dql->prev_num_queued) + dql->prev_ovlimit; dql->slack_start_time = jiffies; dql->lowest_slack = UINT_MAX; } else if (inprogress && prev_inprogress && !all_prev_completed) { /* * Queue was not starved, check if the limit can be decreased. * A decrease is only considered if the queue has been busy in * the whole interval (the check above). * * If there is slack, the amount of execess data queued above * the the amount needed to prevent starvation, the queue limit * can be decreased. To avoid hysteresis we consider the * minimum amount of slack found over several iterations of the * completion routine. */ unsigned int slack, slack_last_objs; /* * Slack is the maximum of * - The queue limit plus previous over-limit minus twice * the number of objects completed. Note that two times * number of completed bytes is a basis for an upper bound * of the limit. * - Portion of objects in the last queuing operation that * was not part of non-zero previous over-limit. That is * "round down" by non-overlimit portion of the last * queueing operation. */ slack = POSDIFF(limit + dql->prev_ovlimit, 2 * (completed - dql->num_completed)); slack_last_objs = dql->prev_ovlimit ? POSDIFF(dql->prev_last_obj_cnt, dql->prev_ovlimit) : 0; slack = max(slack, slack_last_objs); if (slack < dql->lowest_slack) dql->lowest_slack = slack; if (time_after(jiffies, dql->slack_start_time + dql->slack_hold_time)) { limit = POSDIFF(limit, dql->lowest_slack); dql->slack_start_time = jiffies; dql->lowest_slack = UINT_MAX; } } /* Enforce bounds on limit */ limit = clamp(limit, dql->min_limit, dql->max_limit); if (limit != dql->limit) { dql->limit = limit; ovlimit = 0; } dql->adj_limit = limit + completed; dql->prev_ovlimit = ovlimit; dql->prev_last_obj_cnt = dql->last_obj_cnt; dql->num_completed = completed; dql->prev_num_queued = num_queued; }
static inline void dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local) { struct netdev_queue *txq; int cpu, clean_done = 0; u32 txd_free_idx; #if defined (CONFIG_RAETH_BQL) u32 bytes_sent_ge1 = 0; #if defined (CONFIG_PSEUDO_SUPPORT) u32 bytes_sent_ge2 = 0; #endif #endif spin_lock(&ei_local->page_lock); txd_free_idx = ei_local->txd_free_idx; while (clean_done < (NUM_TX_DESC-2)) { struct PDMA_txdesc *txd; struct sk_buff *skb; skb = ei_local->txd_buff[txd_free_idx]; if (!skb) break; txd = &ei_local->txd_ring[txd_free_idx]; /* check TXD not owned by DMA */ if (!(ACCESS_ONCE(txd->txd_info2) & TX2_DMA_DONE)) break; if (skb != (struct sk_buff *)0xFFFFFFFF) { #if defined (CONFIG_RAETH_BQL) #if defined (CONFIG_PSEUDO_SUPPORT) if (skb->dev == ei_local->PseudoDev) bytes_sent_ge2 += skb->len; else #endif bytes_sent_ge1 += skb->len; #endif dev_kfree_skb(skb); } ei_local->txd_buff[txd_free_idx] = NULL; txd_free_idx = (txd_free_idx + 1) % NUM_TX_DESC; clean_done++; } if (ei_local->txd_free_idx != txd_free_idx) ei_local->txd_free_idx = txd_free_idx; spin_unlock(&ei_local->page_lock); if (!clean_done) return; cpu = smp_processor_id(); if (netif_running(dev)) { txq = netdev_get_tx_queue(dev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge1); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #if defined (CONFIG_PSEUDO_SUPPORT) if (netif_running(ei_local->PseudoDev)) { txq = netdev_get_tx_queue(ei_local->PseudoDev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge2); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #endif }
int _ext4_get_encryption_info(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_crypt_info *crypt_info; char full_key_descriptor[EXT4_KEY_DESC_PREFIX_SIZE + (EXT4_KEY_DESCRIPTOR_SIZE * 2) + 1]; struct key *keyring_key = NULL; struct ext4_encryption_key *master_key; struct ext4_encryption_context ctx; struct user_key_payload *ukp; struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct crypto_ablkcipher *ctfm; const char *cipher_str; char raw_key[EXT4_MAX_KEY_SIZE]; char mode; int res; if (!ext4_read_workqueue) { res = ext4_init_crypto(); if (res) return res; } retry: crypt_info = ACCESS_ONCE(ei->i_crypt_info); if (crypt_info) { if (!crypt_info->ci_keyring_key || key_validate(crypt_info->ci_keyring_key) == 0) return 0; ext4_free_encryption_info(inode, crypt_info); goto retry; } res = ext4_xattr_get(inode, EXT4_XATTR_INDEX_ENCRYPTION, EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, &ctx, sizeof(ctx)); if (res < 0) { if (!DUMMY_ENCRYPTION_ENABLED(sbi)) return res; ctx.contents_encryption_mode = EXT4_ENCRYPTION_MODE_AES_256_XTS; ctx.filenames_encryption_mode = EXT4_ENCRYPTION_MODE_AES_256_CTS; ctx.flags = 0; } else if (res != sizeof(ctx)) return -EINVAL; res = 0; crypt_info = kmem_cache_alloc(ext4_crypt_info_cachep, GFP_KERNEL); if (!crypt_info) return -ENOMEM; crypt_info->ci_flags = ctx.flags; crypt_info->ci_data_mode = ctx.contents_encryption_mode; crypt_info->ci_filename_mode = ctx.filenames_encryption_mode; crypt_info->ci_ctfm = NULL; crypt_info->ci_keyring_key = NULL; memcpy(crypt_info->ci_master_key, ctx.master_key_descriptor, sizeof(crypt_info->ci_master_key)); if (S_ISREG(inode->i_mode)) mode = crypt_info->ci_data_mode; else if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) mode = crypt_info->ci_filename_mode; else BUG(); switch (mode) { case EXT4_ENCRYPTION_MODE_AES_256_XTS: cipher_str = "xts(aes)"; break; case EXT4_ENCRYPTION_MODE_AES_256_CTS: cipher_str = "cts(cbc(aes))"; break; default: printk_once(KERN_WARNING "ext4: unsupported key mode %d (ino %u)\n", mode, (unsigned) inode->i_ino); res = -ENOKEY; goto out; } if (DUMMY_ENCRYPTION_ENABLED(sbi)) { memset(raw_key, 0x42, EXT4_AES_256_XTS_KEY_SIZE); goto got_key; } memcpy(full_key_descriptor, EXT4_KEY_DESC_PREFIX, EXT4_KEY_DESC_PREFIX_SIZE); sprintf(full_key_descriptor + EXT4_KEY_DESC_PREFIX_SIZE, "%*phN", EXT4_KEY_DESCRIPTOR_SIZE, ctx.master_key_descriptor); full_key_descriptor[EXT4_KEY_DESC_PREFIX_SIZE + (2 * EXT4_KEY_DESCRIPTOR_SIZE)] = '\0'; keyring_key = request_key(&key_type_logon, full_key_descriptor, NULL); if (IS_ERR(keyring_key)) { res = PTR_ERR(keyring_key); keyring_key = NULL; goto out; } crypt_info->ci_keyring_key = keyring_key; if (keyring_key->type != &key_type_logon) { printk_once(KERN_WARNING "ext4: key type must be logon\n"); res = -ENOKEY; goto out; } ukp = ((struct user_key_payload *)keyring_key->payload.data); if (ukp->datalen != sizeof(struct ext4_encryption_key)) { res = -EINVAL; goto out; } master_key = (struct ext4_encryption_key *)ukp->data; BUILD_BUG_ON(EXT4_AES_128_ECB_KEY_SIZE != EXT4_KEY_DERIVATION_NONCE_SIZE); if (master_key->size != EXT4_AES_256_XTS_KEY_SIZE) { printk_once(KERN_WARNING "ext4: key size incorrect: %d\n", master_key->size); res = -ENOKEY; goto out; } res = ext4_derive_key_aes(ctx.nonce, master_key->raw, raw_key); got_key: ctfm = crypto_alloc_ablkcipher(cipher_str, 0, 0); if (!ctfm || IS_ERR(ctfm)) { res = ctfm ? PTR_ERR(ctfm) : -ENOMEM; printk(KERN_DEBUG "%s: error %d (inode %u) allocating crypto tfm\n", __func__, res, (unsigned) inode->i_ino); goto out; } crypt_info->ci_ctfm = ctfm; crypto_ablkcipher_clear_flags(ctfm, ~0); crypto_tfm_set_flags(crypto_ablkcipher_tfm(ctfm), CRYPTO_TFM_REQ_WEAK_KEY); res = crypto_ablkcipher_setkey(ctfm, raw_key, ext4_encryption_key_size(mode)); if (res) goto out; memzero_explicit(raw_key, sizeof(raw_key)); if (cmpxchg(&ei->i_crypt_info, NULL, crypt_info) != NULL) { ext4_free_crypt_info(crypt_info); goto retry; } return 0; out: if (res == -ENOKEY) res = 0; ext4_free_crypt_info(crypt_info); memzero_explicit(raw_key, sizeof(raw_key)); return res; }
/* must be spinlock protected */ static void fe_dma_init(END_DEVICE *ei_local) { int i; u32 regVal; /* init PDMA TX ring */ for (i = 0; i < NUM_TX_DESC; i++) { struct PDMA_txdesc *txd = &ei_local->txd_ring[i]; ei_local->txd_buff[i] = NULL; ACCESS_ONCE(txd->txd_info1) = 0; ACCESS_ONCE(txd->txd_info2) = TX2_DMA_DONE; #if defined (RAETH_PDMA_V2) ACCESS_ONCE(txd->txd_info4) = 0; #else ACCESS_ONCE(txd->txd_info4) = TX4_DMA_QN(3); #endif ACCESS_ONCE(txd->txd_info3) = 0; } /* init PDMA RX ring */ for (i = 0; i < NUM_RX_DESC; i++) { struct PDMA_rxdesc *rxd = &ei_local->rxd_ring[i]; #if defined (RAETH_PDMA_V2) ACCESS_ONCE(rxd->rxd_info1) = (u32)dma_map_single(NULL, ei_local->rxd_buff[i]->data, MAX_RX_LENGTH + NET_IP_ALIGN, DMA_FROM_DEVICE); ACCESS_ONCE(rxd->rxd_info2) = RX2_DMA_SDL0_SET(MAX_RX_LENGTH); #else ACCESS_ONCE(rxd->rxd_info1) = (u32)dma_map_single(NULL, ei_local->rxd_buff[i]->data, MAX_RX_LENGTH, DMA_FROM_DEVICE); ACCESS_ONCE(rxd->rxd_info2) = RX2_DMA_LS0; #endif ACCESS_ONCE(rxd->rxd_info3) = 0; ACCESS_ONCE(rxd->rxd_info4) = 0; } wmb(); /* clear PDMA */ regVal = sysRegRead(PDMA_GLO_CFG); regVal &= ~(CSR_CLKGATE | RX_DMA_EN | TX_DMA_EN); sysRegWrite(PDMA_GLO_CFG, regVal); /* GDMA1/2 <- TX Ring #0 */ sysRegWrite(TX_BASE_PTR0, phys_to_bus((u32)ei_local->txd_ring_phy)); sysRegWrite(TX_MAX_CNT0, cpu_to_le32(NUM_TX_DESC)); sysRegWrite(TX_CTX_IDX0, 0); sysRegWrite(PDMA_RST_CFG, PST_DTX_IDX0); ei_local->txd_last_idx = le32_to_cpu(sysRegRead(TX_CTX_IDX0)); ei_local->txd_free_idx = ei_local->txd_last_idx; /* GDMA1/2 -> RX Ring #0 */ sysRegWrite(RX_BASE_PTR0, phys_to_bus((u32)ei_local->rxd_ring_phy)); sysRegWrite(RX_MAX_CNT0, cpu_to_le32(NUM_RX_DESC)); sysRegWrite(RX_CALC_IDX0, cpu_to_le32(NUM_RX_DESC - 1)); sysRegWrite(PDMA_RST_CFG, PST_DRX_IDX0); /* only the following chipset need to set it */ #if defined (CONFIG_RALINK_RT3052) || defined (CONFIG_RALINK_RT3883) // set 1us timer count in unit of clock cycle regVal = sysRegRead(FE_GLO_CFG); regVal &= ~(0xff << 8); //clear bit8-bit15 regVal |= (((get_surfboard_sysclk()/1000000)) << 8); sysRegWrite(FE_GLO_CFG, regVal); #endif /* config DLY interrupt */ sysRegWrite(DLY_INT_CFG, FE_DLY_INIT_VALUE); }
/* * Check whether the dentry is still valid * * If the entry validity timeout has expired and the dentry is * positive, try to redo the lookup. If the lookup results in a * different inode, then let the VFS invalidate the dentry and redo * the lookup once more. If the lookup results in the same inode, * then refresh the attributes, timeouts and mark the dentry valid. */ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags) { struct inode *inode; struct dentry *parent; struct fuse_conn *fc; struct fuse_inode *fi; int ret; inode = ACCESS_ONCE(entry->d_inode); if (inode && is_bad_inode(inode)) goto invalid; else if (fuse_dentry_time(entry) < get_jiffies_64()) { int err; struct fuse_entry_out outarg; struct fuse_req *req; struct fuse_forget_link *forget; u64 attr_version; /* For negative dentries, always do a fresh lookup */ if (!inode) goto invalid; ret = -ECHILD; if (flags & LOOKUP_RCU) goto out; fc = get_fuse_conn(inode); req = fuse_get_req_nopages(fc); ret = PTR_ERR(req); if (IS_ERR(req)) goto out; forget = fuse_alloc_forget(); if (!forget) { fuse_put_request(fc, req); ret = -ENOMEM; goto out; } attr_version = fuse_get_attr_version(fc); parent = dget_parent(entry); fuse_lookup_init(fc, req, get_node_id(parent->d_inode), &entry->d_name, &outarg); fuse_request_send(fc, req); dput(parent); err = req->out.h.error; fuse_put_request(fc, req); /* Zero nodeid is same as -ENOENT */ if (!err && !outarg.nodeid) err = -ENOENT; if (!err) { fi = get_fuse_inode(inode); if (outarg.nodeid != get_node_id(inode)) { fuse_queue_forget(fc, forget, outarg.nodeid, 1); goto invalid; } spin_lock(&fc->lock); fi->nlookup++; spin_unlock(&fc->lock); } kfree(forget); if (err || (outarg.attr.mode ^ inode->i_mode) & S_IFMT) goto invalid; fuse_change_attributes(inode, &outarg.attr, entry_attr_timeout(&outarg), attr_version); fuse_change_entry_timeout(entry, &outarg); } else if (inode) { fi = get_fuse_inode(inode); if (flags & LOOKUP_RCU) { if (test_bit(FUSE_I_INIT_RDPLUS, &fi->state)) return -ECHILD; } else if (test_and_clear_bit(FUSE_I_INIT_RDPLUS, &fi->state)) { parent = dget_parent(entry); fuse_advise_use_readdirplus(parent->d_inode); dput(parent); } } ret = 1; out: return ret; invalid: ret = 0; if (!(flags & LOOKUP_RCU) && check_submounts_and_drop(entry) != 0) ret = 1; goto out; }
/* This routine provides PHY access routines for * mdio clause45 . */ static int octnet_mdio45_access(struct lio *lio, int op, int loc, int *value) { struct octeon_device *oct_dev = lio->oct_dev; struct octeon_soft_command *sc; struct oct_mdio_cmd_resp *mdio_cmd_rsp; struct oct_mdio_cmd_context *mdio_cmd_ctx; struct oct_mdio_cmd *mdio_cmd; int retval = 0; sc = (struct octeon_soft_command *) octeon_alloc_soft_command(oct_dev, sizeof(struct oct_mdio_cmd), sizeof(struct oct_mdio_cmd_resp), sizeof(struct oct_mdio_cmd_context)); if (!sc) return -ENOMEM; mdio_cmd_ctx = (struct oct_mdio_cmd_context *)sc->ctxptr; mdio_cmd_rsp = (struct oct_mdio_cmd_resp *)sc->virtrptr; mdio_cmd = (struct oct_mdio_cmd *)sc->virtdptr; ACCESS_ONCE(mdio_cmd_ctx->cond) = 0; mdio_cmd_ctx->octeon_id = lio_get_device_id(oct_dev); mdio_cmd->op = op; mdio_cmd->mdio_addr = loc; if (op) mdio_cmd->value1 = *value; octeon_swap_8B_data((u64 *)mdio_cmd, sizeof(struct oct_mdio_cmd) / 8); sc->iq_no = lio->linfo.txpciq[0].s.q_no; octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC, OPCODE_NIC_MDIO45, 0, 0, 0); sc->wait_time = 1000; sc->callback = octnet_mdio_resp_callback; sc->callback_arg = sc; init_waitqueue_head(&mdio_cmd_ctx->wc); retval = octeon_send_soft_command(oct_dev, sc); if (retval == IQ_SEND_FAILED) { dev_err(&oct_dev->pci_dev->dev, "octnet_mdio45_access instruction failed status: %x\n", retval); retval = -EBUSY; } else { /* Sleep on a wait queue till the cond flag indicates that the * response arrived */ sleep_cond(&mdio_cmd_ctx->wc, &mdio_cmd_ctx->cond); retval = mdio_cmd_rsp->status; if (retval) { dev_err(&oct_dev->pci_dev->dev, "octnet mdio45 access failed\n"); retval = -EBUSY; } else { octeon_swap_8B_data((u64 *)(&mdio_cmd_rsp->resp), sizeof(struct oct_mdio_cmd) / 8); if (ACCESS_ONCE(mdio_cmd_ctx->cond) == 1) { if (!op) *value = mdio_cmd_rsp->resp.value1; } else { retval = -EINVAL; } } } octeon_free_soft_command(oct_dev, sc); return retval; }
static inline __u16 read_avail_idx(struct mic_vring *vr) { return ACCESS_ONCE(vr->info->avail_idx); }
static ssize_t wdm_read (struct file *file, char __user *buffer, size_t count, loff_t *ppos) { int rv, cntr; int i = 0; struct wdm_device *desc = file->private_data; rv = mutex_lock_interruptible(&desc->rlock); /*concurrent reads */ if (rv < 0) return -ERESTARTSYS; cntr = ACCESS_ONCE(desc->length); if (cntr == 0) { desc->read = 0; retry: if (test_bit(WDM_DISCONNECTING, &desc->flags)) { rv = -ENODEV; goto err; } if (test_bit(WDM_OVERFLOW, &desc->flags)) { clear_bit(WDM_OVERFLOW, &desc->flags); rv = -ENOBUFS; goto err; } i++; if (file->f_flags & O_NONBLOCK) { if (!test_bit(WDM_READ, &desc->flags)) { rv = cntr ? cntr : -EAGAIN; goto err; } rv = 0; } else { rv = wait_event_interruptible(desc->wait, test_bit(WDM_READ, &desc->flags)); } /* may have happened while we slept */ if (test_bit(WDM_DISCONNECTING, &desc->flags)) { rv = -ENODEV; goto err; } usb_mark_last_busy(interface_to_usbdev(desc->intf)); if (rv < 0) { rv = -ERESTARTSYS; goto err; } spin_lock_irq(&desc->iuspin); if (desc->rerr) { /* read completed, error happened */ desc->rerr = 0; spin_unlock_irq(&desc->iuspin); rv = -EIO; goto err; } /* * recheck whether we've lost the race * against the completion handler */ if (!test_bit(WDM_READ, &desc->flags)) { /* lost race */ spin_unlock_irq(&desc->iuspin); goto retry; } if (!desc->reslength) { /* zero length read */ spin_unlock_irq(&desc->iuspin); goto retry; } cntr = desc->length; spin_unlock_irq(&desc->iuspin); } if (cntr > count) cntr = count; rv = copy_to_user(buffer, desc->ubuf, cntr); if (rv > 0) { rv = -EFAULT; goto err; } spin_lock_irq(&desc->iuspin); for (i = 0; i < desc->length - cntr; i++) desc->ubuf[i] = desc->ubuf[i + cntr]; desc->length -= cntr; /* in case we had outstanding data */ if (!desc->length) clear_bit(WDM_READ, &desc->flags); spin_unlock_irq(&desc->iuspin); rv = cntr; err: mutex_unlock(&desc->rlock); return rv; }
/* Deferred service handler, run as interrupt-fired tasklet */ static void caam_jr_dequeue(unsigned long devarg) { int hw_idx, sw_idx, i, head, tail; struct device *dev = (struct device *)devarg; struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); u32 *userdesc, userstatus; void *userarg; while (rd_reg32(&jrp->rregs->outring_used)) { head = ACCESS_ONCE(jrp->head); spin_lock(&jrp->outlock); sw_idx = tail = jrp->tail; hw_idx = jrp->out_ring_read_index; for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { sw_idx = (tail + i) & (JOBR_DEPTH - 1); smp_read_barrier_depends(); if (jrp->outring[hw_idx].desc == jrp->entinfo[sw_idx].desc_addr_dma) break; /* found */ } /* we should never fail to find a matching descriptor */ BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0); /* Unmap just-run descriptor so we can post-process */ dma_unmap_single(dev, jrp->outring[hw_idx].desc, jrp->entinfo[sw_idx].desc_size, DMA_TO_DEVICE); /* mark completed, avoid matching on a recycled desc addr */ jrp->entinfo[sw_idx].desc_addr_dma = 0; /* Stash callback params for use outside of lock */ usercall = jrp->entinfo[sw_idx].callbk; userarg = jrp->entinfo[sw_idx].cbkarg; userdesc = jrp->entinfo[sw_idx].desc_addr_virt; userstatus = jrp->outring[hw_idx].jrstatus; /* set done */ wr_reg32(&jrp->rregs->outring_rmvd, 1); jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & (JOBR_DEPTH - 1); /* * if this job completed out-of-order, do not increment * the tail. Otherwise, increment tail by 1 plus the * number of subsequent jobs already completed out-of-order */ if (sw_idx == tail) { do { tail = (tail + 1) & (JOBR_DEPTH - 1); smp_read_barrier_depends(); } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && jrp->entinfo[tail].desc_addr_dma == 0); jrp->tail = tail; } spin_unlock(&jrp->outlock); /* Finally, execute user's callback */ usercall(dev, userdesc, userstatus, userarg); } /* reenable / unmask IRQs */ clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); }
/* * Check whether the dentry is still valid * * If the entry validity timeout has expired and the dentry is * positive, try to redo the lookup. If the lookup results in a * different inode, then let the VFS invalidate the dentry and redo * the lookup once more. If the lookup results in the same inode, * then refresh the attributes, timeouts and mark the dentry valid. */ static int fuse_dentry_revalidate(struct dentry *entry, struct nameidata *nd) { struct inode *inode; inode = ACCESS_ONCE(entry->d_inode); if (inode && is_bad_inode(inode)) return 0; else if (fuse_dentry_time(entry) < get_jiffies_64()) { int err; struct fuse_entry_out outarg; struct fuse_conn *fc; struct fuse_req *req; struct fuse_forget_link *forget; struct dentry *parent; u64 attr_version; /* For negative dentries, always do a fresh lookup */ if (!inode) return 0; if (nd && (nd->flags & LOOKUP_RCU)) return -ECHILD; fc = get_fuse_conn(inode); req = fuse_get_req(fc); if (IS_ERR(req)) return 0; forget = fuse_alloc_forget(); if (!forget) { fuse_put_request(fc, req); return 0; } attr_version = fuse_get_attr_version(fc); parent = dget_parent(entry); fuse_lookup_init(fc, req, get_node_id(parent->d_inode), &entry->d_name, &outarg); fuse_request_send(fc, req); dput(parent); err = req->out.h.error; fuse_put_request(fc, req); /* Zero nodeid is same as -ENOENT */ if (!err && !outarg.nodeid) err = -ENOENT; if (!err) { struct fuse_inode *fi = get_fuse_inode(inode); if (outarg.nodeid != get_node_id(inode)) { fuse_queue_forget(fc, forget, outarg.nodeid, 1); return 0; } spin_lock(&fc->lock); fi->nlookup++; spin_unlock(&fc->lock); } kfree(forget); if (err || (outarg.attr.mode ^ inode->i_mode) & S_IFMT) return 0; fuse_change_attributes(inode, &outarg.attr, entry_attr_timeout(&outarg), attr_version); fuse_change_entry_timeout(entry, &outarg); } return 1; }
int ovl_permission(struct inode *inode, int mask) { struct ovl_entry *oe; struct dentry *alias = NULL; struct inode *realinode; struct dentry *realdentry; bool is_upper; int err; if (S_ISDIR(inode->i_mode)) { oe = inode->i_private; } else if (mask & MAY_NOT_BLOCK) { return -ECHILD; } else { /* * For non-directories find an alias and get the info * from there. */ spin_lock(&inode->i_lock); if (WARN_ON(list_empty(&inode->i_dentry))) { spin_unlock(&inode->i_lock); return -ENOENT; } alias = list_entry(inode->i_dentry.next, struct dentry, d_alias); dget(alias); spin_unlock(&inode->i_lock); oe = alias->d_fsdata; } realdentry = ovl_entry_real(oe, &is_upper); /* Careful in RCU walk mode */ realinode = ACCESS_ONCE(realdentry->d_inode); if (!realinode) { WARN_ON(!(mask & MAY_NOT_BLOCK)); err = -ENOENT; goto out_dput; } if (mask & MAY_WRITE) { umode_t mode = realinode->i_mode; /* * Writes will always be redirected to upper layer, so * ignore lower layer being read-only. * * If the overlay itself is read-only then proceed * with the permission check, don't return EROFS. * This will only happen if this is the lower layer of * another overlayfs. * * If upper fs becomes read-only after the overlay was * constructed return EROFS to prevent modification of * upper layer. */ err = -EROFS; if (is_upper && !IS_RDONLY(inode) && IS_RDONLY(realinode) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) goto out_dput; /* * Nobody gets write access to an immutable file. */ err = -EACCES; if (IS_IMMUTABLE(realinode)) goto out_dput; } if (realinode->i_op->permission) err = realinode->i_op->permission(realinode, mask); else err = generic_permission(realinode, mask); out_dput: dput(alias); return err; }
/* * Fully allocate the p2m structure for a given pfn. We need to check * that both the top and mid levels are allocated, and make sure the * parallel mfn tree is kept in sync. We may race with other cpus, so * the new pages are installed with cmpxchg; if we lose the race then * simply free the page we allocated and use the one that's there. */ static bool alloc_p2m(unsigned long pfn) { unsigned topidx, mididx; unsigned long *top_mfn_p, *mid_mfn; pte_t *ptep, *pte_pg; unsigned int level; unsigned long flags; unsigned long addr = (unsigned long)(xen_p2m_addr + pfn); unsigned long p2m_pfn; topidx = p2m_top_index(pfn); mididx = p2m_mid_index(pfn); ptep = lookup_address(addr, &level); BUG_ON(!ptep || level != PG_LEVEL_4K); pte_pg = (pte_t *)((unsigned long)ptep & ~(PAGE_SIZE - 1)); if (pte_pg == p2m_missing_pte || pte_pg == p2m_identity_pte) { /* PMD level is missing, allocate a new one */ ptep = alloc_p2m_pmd(addr, pte_pg); if (!ptep) return false; } if (p2m_top_mfn) { top_mfn_p = &p2m_top_mfn[topidx]; mid_mfn = ACCESS_ONCE(p2m_top_mfn_p[topidx]); BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p); if (mid_mfn == p2m_mid_missing_mfn) { /* Separately check the mid mfn level */ unsigned long missing_mfn; unsigned long mid_mfn_mfn; unsigned long old_mfn; mid_mfn = alloc_p2m_page(); if (!mid_mfn) return false; p2m_mid_mfn_init(mid_mfn, p2m_missing); missing_mfn = virt_to_mfn(p2m_mid_missing_mfn); mid_mfn_mfn = virt_to_mfn(mid_mfn); old_mfn = cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn); if (old_mfn != missing_mfn) { free_p2m_page(mid_mfn); mid_mfn = mfn_to_virt(old_mfn); } else { p2m_top_mfn_p[topidx] = mid_mfn; } } } else { mid_mfn = NULL; } p2m_pfn = pte_pfn(READ_ONCE(*ptep)); if (p2m_pfn == PFN_DOWN(__pa(p2m_identity)) || p2m_pfn == PFN_DOWN(__pa(p2m_missing))) { /* p2m leaf page is missing */ unsigned long *p2m; p2m = alloc_p2m_page(); if (!p2m) return false; if (p2m_pfn == PFN_DOWN(__pa(p2m_missing))) p2m_init(p2m); else p2m_init_identity(p2m, pfn & ~(P2M_PER_PAGE - 1)); spin_lock_irqsave(&p2m_update_lock, flags); if (pte_pfn(*ptep) == p2m_pfn) { set_pte(ptep, pfn_pte(PFN_DOWN(__pa(p2m)), PAGE_KERNEL)); if (mid_mfn) mid_mfn[mididx] = virt_to_mfn(p2m); p2m = NULL; } spin_unlock_irqrestore(&p2m_update_lock, flags); if (p2m) free_p2m_page(p2m); } return true; }
int ovl_permission(struct inode *inode, int mask) { struct ovl_entry *oe; struct dentry *alias = NULL; struct inode *realinode; struct dentry *realdentry; bool is_upper; int err; if (S_ISDIR(inode->i_mode)) { oe = inode->i_private; } else if (mask & MAY_NOT_BLOCK) { return -ECHILD; } else { /* * For non-directories find an alias and get the info * from there. */ alias = d_find_any_alias(inode); if (WARN_ON(!alias)) return -ENOENT; oe = alias->d_fsdata; } realdentry = ovl_entry_real(oe, &is_upper); if (ovl_is_default_permissions(inode)) { struct kstat stat; struct path realpath = { .dentry = realdentry }; if (mask & MAY_NOT_BLOCK) return -ECHILD; realpath.mnt = ovl_entry_mnt_real(oe, inode, is_upper); err = vfs_getattr(&realpath, &stat); if (err) goto out_dput; err = -ESTALE; if ((stat.mode ^ inode->i_mode) & S_IFMT) goto out_dput; inode->i_mode = stat.mode; inode->i_uid = stat.uid; inode->i_gid = stat.gid; err = generic_permission(inode, mask); goto out_dput; } /* Careful in RCU walk mode */ realinode = ACCESS_ONCE(realdentry->d_inode); if (!realinode) { WARN_ON(!(mask & MAY_NOT_BLOCK)); err = -ENOENT; goto out_dput; } if (mask & MAY_WRITE) { umode_t mode = realinode->i_mode; /* * Writes will always be redirected to upper layer, so * ignore lower layer being read-only. * * If the overlay itself is read-only then proceed * with the permission check, don't return EROFS. * This will only happen if this is the lower layer of * another overlayfs. * * If upper fs becomes read-only after the overlay was * constructed return EROFS to prevent modification of * upper layer. */ err = -EROFS; if (is_upper && !IS_RDONLY(inode) && IS_RDONLY(realinode) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) goto out_dput; } err = __inode_permission(realinode, mask); out_dput: dput(alias); return err; }
static int rcu_gp_in_progress(struct rcu_state *rsp) { return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); }
int cps_pm_enter_state(enum cps_pm_state state) { unsigned cpu = smp_processor_id(); unsigned core = current_cpu_data.core; unsigned online, left; cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled); u32 *core_ready_count, *nc_core_ready_count; void *nc_addr; cps_nc_entry_fn entry; struct core_boot_config *core_cfg; struct vpe_boot_config *vpe_cfg; /* Check that there is an entry function for this state */ entry = per_cpu(nc_asm_enter, core)[state]; if (!entry) return -EINVAL; /* Calculate which coupled CPUs (VPEs) are online */ #ifdef CONFIG_MIPS_MT if (cpu_online(cpu)) { cpumask_and(coupled_mask, cpu_online_mask, &cpu_sibling_map[cpu]); online = cpumask_weight(coupled_mask); cpumask_clear_cpu(cpu, coupled_mask); } else #endif { cpumask_clear(coupled_mask); online = 1; } /* Setup the VPE to run mips_cps_pm_restore when started again */ if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { /* Power gating relies upon CPS SMP */ if (!mips_cps_smp_in_use()) return -EINVAL; core_cfg = &mips_cps_core_bootcfg[core]; vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(¤t_cpu_data)]; vpe_cfg->pc = (unsigned long)mips_cps_pm_restore; vpe_cfg->gp = (unsigned long)current_thread_info(); vpe_cfg->sp = 0; } /* Indicate that this CPU might not be coherent */ cpumask_clear_cpu(cpu, &cpu_coherent_mask); smp_mb__after_atomic(); /* Create a non-coherent mapping of the core ready_count */ core_ready_count = per_cpu(ready_count, core); nc_addr = kmap_noncoherent(virt_to_page(core_ready_count), (unsigned long)core_ready_count); nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK); nc_core_ready_count = nc_addr; /* Ensure ready_count is zero-initialised before the assembly runs */ ACCESS_ONCE(*nc_core_ready_count) = 0; coupled_barrier(&per_cpu(pm_barrier, core), online); /* Run the generated entry code */ left = entry(online, nc_core_ready_count); /* Remove the non-coherent mapping of ready_count */ kunmap_noncoherent(); /* Indicate that this CPU is definitely coherent */ cpumask_set_cpu(cpu, &cpu_coherent_mask); /* * If this VPE is the first to leave the non-coherent wait state then * it needs to wake up any coupled VPEs still running their wait * instruction so that they return to cpuidle, which can then complete * coordination between the coupled VPEs & provide the governor with * a chance to reflect on the length of time the VPEs were in the * idle state. */ if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online)) arch_send_call_function_ipi_mask(coupled_mask); return 0; }
/* * Process the rx status descriptor on 5212 */ static int ath5k_hw_proc_5212_rx_status(struct ath5k_hw *ah, struct ath5k_desc *desc, struct ath5k_rx_status *rs) { struct ath5k_hw_rx_status *rx_status; u32 rxstat0, rxstat1; rx_status = &desc->ud.ds_rx.rx_stat; rxstat1 = ACCESS_ONCE(rx_status->rx_status_1); /* No frame received / not ready */ if (unlikely(!(rxstat1 & AR5K_5212_RX_DESC_STATUS1_DONE))) return -EINPROGRESS; memset(rs, 0, sizeof(struct ath5k_rx_status)); rxstat0 = ACCESS_ONCE(rx_status->rx_status_0); /* * Frame receive status */ rs->rs_datalen = rxstat0 & AR5K_5212_RX_DESC_STATUS0_DATA_LEN; rs->rs_rssi = AR5K_REG_MS(rxstat0, AR5K_5212_RX_DESC_STATUS0_RECEIVE_SIGNAL); rs->rs_rate = AR5K_REG_MS(rxstat0, AR5K_5212_RX_DESC_STATUS0_RECEIVE_RATE); rs->rs_antenna = AR5K_REG_MS(rxstat0, AR5K_5212_RX_DESC_STATUS0_RECEIVE_ANTENNA); rs->rs_more = !!(rxstat0 & AR5K_5212_RX_DESC_STATUS0_MORE); rs->rs_tstamp = AR5K_REG_MS(rxstat1, AR5K_5212_RX_DESC_STATUS1_RECEIVE_TIMESTAMP); /* * Key table status */ if (rxstat1 & AR5K_5212_RX_DESC_STATUS1_KEY_INDEX_VALID) rs->rs_keyix = AR5K_REG_MS(rxstat1, AR5K_5212_RX_DESC_STATUS1_KEY_INDEX); else rs->rs_keyix = AR5K_RXKEYIX_INVALID; /* * Receive/descriptor errors */ if (!(rxstat1 & AR5K_5212_RX_DESC_STATUS1_FRAME_RECEIVE_OK)) { if (rxstat1 & AR5K_5212_RX_DESC_STATUS1_CRC_ERROR) rs->rs_status |= AR5K_RXERR_CRC; if (rxstat1 & AR5K_5212_RX_DESC_STATUS1_PHY_ERROR) { rs->rs_status |= AR5K_RXERR_PHY; rs->rs_phyerr = AR5K_REG_MS(rxstat1, AR5K_5212_RX_DESC_STATUS1_PHY_ERROR_CODE); if (!ah->ah_capabilities.cap_has_phyerr_counters) ath5k_ani_phy_error_report(ah, rs->rs_phyerr); } if (rxstat1 & AR5K_5212_RX_DESC_STATUS1_DECRYPT_CRC_ERROR) rs->rs_status |= AR5K_RXERR_DECRYPT; if (rxstat1 & AR5K_5212_RX_DESC_STATUS1_MIC_ERROR) rs->rs_status |= AR5K_RXERR_MIC; } return 0; }