static void get_data_point(void) { struct scbs_data_point item; unsigned long head; unsigned long tail; //Caller must check that data is present spin_lock(&consumer_lock); head = ACCESS_ONCE(buffer.head); tail = buffer.tail; if (CIRC_CNT(head, tail, buffer.size) >= 1) { /* read index before reading contents at that index */ smp_read_barrier_depends(); /* extract one item from the buffer */ item = buffer.buf[tail]; scbs_data_point = item; smp_mb(); /* finish reading descriptor before incrementing tail */ buffer.tail = (tail + 1) & (buffer.size - 1); } spin_unlock(&consumer_lock); }
/** * seccomp_run_filters - evaluates all seccomp filters against @syscall * @syscall: number of the current system call * * Returns valid seccomp BPF response codes. */ static u32 seccomp_run_filters(struct seccomp_data *sd) { struct seccomp_filter *f = ACCESS_ONCE(current->seccomp.filter); struct seccomp_data sd_local; u32 ret = SECCOMP_RET_ALLOW; /* Ensure unexpected behavior doesn't result in failing open. */ if (unlikely(WARN_ON(f == NULL))) return SECCOMP_RET_KILL; /* Make sure cross-thread synced filter points somewhere sane. */ smp_read_barrier_depends(); if (!sd) { populate_seccomp_data(&sd_local); sd = &sd_local; } /* * All filters in the list are evaluated and the lowest BPF return * value always takes priority (ignoring the DATA). */ for (; f; f = f->prev) { u32 cur_ret = BPF_PROG_RUN(f->prog, (void *)sd); if ((cur_ret & SECCOMP_RET_ACTION) < (ret & SECCOMP_RET_ACTION)) ret = cur_ret; } return ret; }
int set_task_ioprio(struct task_struct *task, int ioprio) { #ifdef CONFIG_IOSCHED_BFQ int err, i; #else int err; #endif struct io_context *ioc; const struct cred *cred = current_cred(), *tcred; rcu_read_lock(); tcred = __task_cred(task); if (tcred->uid != cred->euid && tcred->uid != cred->uid && !capable(CAP_SYS_NICE)) { rcu_read_unlock(); return -EPERM; } rcu_read_unlock(); err = security_task_setioprio(task, ioprio); if (err) return err; task_lock(task); do { ioc = task->io_context; /* see wmb() in current_io_context() */ smp_read_barrier_depends(); if (ioc) break; ioc = alloc_io_context(GFP_ATOMIC, -1); if (!ioc) { err = -ENOMEM; break; } #ifdef CONFIG_IOSCHED_BFQ /* let other ioc users see the new values */ smp_wmb(); #endif task->io_context = ioc; } while (1); if (!err) { ioc->ioprio = ioprio; #ifdef CONFIG_IOSCHED_BFQ /* make sure schedulers see the new ioprio value */ wmb(); for (i = 0; i < IOC_IOPRIO_CHANGED_BITS; i++) set_bit(i, ioc->ioprio_changed); #else ioc->ioprio_changed = 1; #endif } task_unlock(task); return err; }
static struct dentry *ovl_upperdentry_dereference(struct ovl_entry *oe) { struct dentry *upperdentry = ACCESS_ONCE(oe->__upperdentry); /* * Make sure to order reads to upperdentry wrt ovl_dentry_update() */ smp_read_barrier_depends(); return upperdentry; }
void trusty_fiq_handler(struct pt_regs *regs, void *svc_sp) { struct fiq_glue_handler *handler; for (handler = ACCESS_ONCE(fiq_handlers); handler; handler = ACCESS_ONCE(handler->next)) { /* Barrier paired with smp_wmb in fiq_glue_register_handler */ smp_read_barrier_depends(); handler->fiq(handler, regs, svc_sp); } }
struct pfq_sock * pfq_get_sock_by_id(pfq_id_t id) { struct pfq_sock *so; if (unlikely((__force int)id >= Q_MAX_ID)) { pr_devel("[PFQ] pfq_get_sock_by_id: bad id=%d!\n", id); return NULL; } so = (struct pfq_sock *)atomic_long_read(&pfq_sock_vector[(__force int)id]); smp_read_barrier_depends(); return so; }
/** * qp_get_savail - return number of avail send entries * * @qp - the qp * * This assumes the s_hlock is held but the s_last * qp variable is uncontrolled. */ static inline u32 qp_get_savail(struct rvt_qp *qp) { u32 slast; u32 ret; smp_read_barrier_depends(); /* see rc.c */ slast = ACCESS_ONCE(qp->s_last); if (qp->s_head >= slast) ret = qp->s_size - (qp->s_head - slast); else ret = slast - qp->s_head; return ret - 1; }
struct mpls_ilm* mpls_get_ilm (unsigned int key) { struct mpls_ilm *ilm = NULL; rcu_read_lock(); ilm = radix_tree_lookup (&mpls_ilm_tree,key); smp_read_barrier_depends(); if (likely(ilm)) mpls_ilm_hold(ilm); rcu_read_unlock(); return ilm; }
/* * * hv_get_ringbuffer_availbytes() * * Get number of bytes available to read and to write to * for the specified ring buffer */ static inline void hv_get_ringbuffer_availbytes(struct hv_ring_buffer_info *rbi, u32 *read, u32 *write) { u32 read_loc, write_loc; smp_read_barrier_depends(); /* Capture the read/write indices before they changed */ read_loc = rbi->ring_buffer->read_index; write_loc = rbi->ring_buffer->write_index; *write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->ring_datasize); *read = rbi->ring_datasize - *write; }
struct mpls_nhlfe* mpls_get_nhlfe (unsigned int key) { struct mpls_nhlfe *nhlfe = NULL; rcu_read_lock(); nhlfe = radix_tree_lookup (&mpls_nhlfe_tree, key); smp_read_barrier_depends(); if (likely(nhlfe)) { mpls_nhlfe_hold(nhlfe); } rcu_read_unlock(); return nhlfe; }
int set_task_ioprio(struct task_struct *task, int ioprio) { int err, i; struct io_context *ioc; const struct cred *cred = current_cred(), *tcred; rcu_read_lock(); tcred = __task_cred(task); if (tcred->uid != cred->euid && tcred->uid != cred->uid && !capable(CAP_SYS_NICE)) { rcu_read_unlock(); return -EPERM; } rcu_read_unlock(); err = security_task_setioprio(task, ioprio); if (err) return err; task_lock(task); do { ioc = task->io_context; /* see wmb() in current_io_context() */ smp_read_barrier_depends(); if (ioc) break; ioc = alloc_io_context(GFP_ATOMIC, -1); if (!ioc) { err = -ENOMEM; break; } smp_wmb(); task->io_context = ioc; } while (1); if (!err) { ioc->ioprio = ioprio; wmb(); for (i = 0; i < IOC_IOPRIO_CHANGED_BITS; i++) set_bit(i, ioc->ioprio_changed); } task_unlock(task); return err; }
/* Multiple occurrences of aio_bh_poll cannot be called concurrently */ int aio_bh_poll(AioContext *ctx) { QEMUBH *bh, **bhp, *next; int ret; ctx->walking_bh++; ret = 0; for (bh = ctx->first_bh; bh; bh = next) { /* Make sure that fetching bh happens before accessing its members */ smp_read_barrier_depends(); next = bh->next; if (!bh->deleted && bh->scheduled) { bh->scheduled = 0; /* Paired with write barrier in bh schedule to ensure reading for * idle & callbacks coming after bh's scheduling. */ smp_rmb(); if (!bh->idle) ret = 1; bh->idle = 0; bh->cb(bh->opaque); } } ctx->walking_bh--; /* remove deleted bhs */ if (!ctx->walking_bh) { qemu_mutex_lock(&ctx->bh_lock); bhp = &ctx->first_bh; while (*bhp) { bh = *bhp; if (bh->deleted) { *bhp = bh->next; g_free(bh); } else { bhp = &bh->next; } } qemu_mutex_unlock(&ctx->bh_lock); } return ret; }
/** * task_work_cancel - cancel a pending work added by task_work_add() * @task: the task which should execute the work * @func: identifies the work to remove * * Find the last queued pending work with ->func == @func and remove * it from queue. * * RETURNS: * The found work or NULL if not found. */ struct callback_head * task_work_cancel(struct task_struct *task, task_work_func_t func) { struct callback_head **pprev = &task->task_works; struct callback_head *work; unsigned long flags; /* * If cmpxchg() fails we continue without updating pprev. * Either we raced with task_work_add() which added the * new entry before this work, we will find it again. Or * we raced with task_work_run(), *pprev == NULL/exited. */ raw_spin_lock_irqsave(&task->pi_lock, flags); while ((work = ACCESS_ONCE(*pprev))) { smp_read_barrier_depends(); if (work->func != func) pprev = &work->next; else if (cmpxchg(pprev, work, work->next) == work) break; } raw_spin_unlock_irqrestore(&task->pi_lock, flags); return work; }
static struct dentry *ovl_upperdentry_dereference(struct ovl_entry *oe) { struct dentry *upperdentry = ACCESS_ONCE(oe->__upperdentry); smp_read_barrier_depends(); return upperdentry; }
/* Deferred service handler, run as interrupt-fired tasklet */ static void caam_jr_dequeue(unsigned long devarg) { int hw_idx, sw_idx, i, head, tail; struct device *dev = (struct device *)devarg; struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); u32 *userdesc, userstatus; dma_addr_t outbusaddr; void *userarg; unsigned long flags; outbusaddr = rd_reg64(&jrp->rregs->outring_base); dma_sync_single_for_cpu(dev, outbusaddr, sizeof(struct jr_outentry) * JOBR_DEPTH, DMA_FROM_DEVICE); spin_lock_irqsave(&jrp->outlock, flags); head = ACCESS_ONCE(jrp->head); sw_idx = tail = jrp->tail; while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && rd_reg32(&jrp->rregs->outring_used)) { hw_idx = jrp->out_ring_read_index; for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { sw_idx = (tail + i) & (JOBR_DEPTH - 1); smp_read_barrier_depends(); if (jrp->outring[hw_idx].desc == jrp->entinfo[sw_idx].desc_addr_dma) break; /* found */ } /* we should never fail to find a matching descriptor */ BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0); /* Unmap just-run descriptor so we can post-process */ dma_unmap_single(dev, jrp->outring[hw_idx].desc, jrp->entinfo[sw_idx].desc_size, DMA_TO_DEVICE); /* mark completed, avoid matching on a recycled desc addr */ jrp->entinfo[sw_idx].desc_addr_dma = 0; /* Stash callback params for use outside of lock */ usercall = jrp->entinfo[sw_idx].callbk; userarg = jrp->entinfo[sw_idx].cbkarg; userdesc = jrp->entinfo[sw_idx].desc_addr_virt; userstatus = jrp->outring[hw_idx].jrstatus; smp_mb(); jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & (JOBR_DEPTH - 1); /* * if this job completed out-of-order, do not increment * the tail. Otherwise, increment tail by 1 plus the * number of subsequent jobs already completed out-of-order */ if (sw_idx == tail) { do { tail = (tail + 1) & (JOBR_DEPTH - 1); smp_read_barrier_depends(); } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && jrp->entinfo[tail].desc_addr_dma == 0); jrp->tail = tail; } /* set done */ wr_reg32(&jrp->rregs->outring_rmvd, 1); spin_unlock_irqrestore(&jrp->outlock, flags); /* Finally, execute user's callback */ usercall(dev, userdesc, userstatus, userarg); spin_lock_irqsave(&jrp->outlock, flags); head = ACCESS_ONCE(jrp->head); sw_idx = tail = jrp->tail; } spin_unlock_irqrestore(&jrp->outlock, flags); /* reenable / unmask IRQs */ clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); }
static int uts_arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) { struct uts_namespace *uts_ns = current->nsproxy->uts_ns; struct ve_struct *ve = get_exec_env(); int i, n1, n2, n3, new_version; struct page **new_pages, **p; /* * For node or in case we've not changed UTS simply * map preallocated original vDSO. * * In turn if we already allocated one for this UTS * simply reuse it. It improves speed significantly. */ if (uts_ns == &init_uts_ns) goto map_init_uts; /* * Dirty lockless hack. Strictly speaking * we need to return @p here if it's non-nil, * but since there only one trasition possible * { =0 ; !=0 } we simply return @uts_ns->vdso.pages */ p = ACCESS_ONCE(uts_ns->vdso.pages); smp_read_barrier_depends(); if (p) goto map_uts; if (sscanf(uts_ns->name.release, "%d.%d.%d", &n1, &n2, &n3) == 3) { /* * If there were no changes on version simply reuse * preallocated one. */ new_version = KERNEL_VERSION(n1, n2, n3); if (new_version == LINUX_VERSION_CODE) goto map_init_uts; } else { /* * If admin is passed malformed string here * lets warn him once but continue working * not using vDSO virtualization at all. It's * better than walk out with error. */ pr_warn_once("Wrong release uts name format detected." " Ignoring vDSO virtualization.\n"); goto map_init_uts; } mutex_lock(&vdso_mutex); if (uts_ns->vdso.pages) { mutex_unlock(&vdso_mutex); goto map_uts; } uts_ns->vdso.nr_pages = init_uts_ns.vdso.nr_pages; uts_ns->vdso.size = init_uts_ns.vdso.size; uts_ns->vdso.version_off= init_uts_ns.vdso.version_off; new_pages = kmalloc(sizeof(struct page *) * init_uts_ns.vdso.nr_pages, GFP_KERNEL); if (!new_pages) { pr_err("Can't allocate vDSO pages array for VE %d\n", ve->veid); goto out_unlock; } for (i = 0; i < uts_ns->vdso.nr_pages; i++) { struct page *p = alloc_page(GFP_KERNEL); if (!p) { pr_err("Can't allocate page for VE %d\n", ve->veid); for (; i > 0; i--) put_page(new_pages[i - 1]); kfree(new_pages); goto out_unlock; } new_pages[i] = p; copy_page(page_address(p), page_address(init_uts_ns.vdso.pages[i])); } uts_ns->vdso.addr = vmap(new_pages, uts_ns->vdso.nr_pages, 0, PAGE_KERNEL); if (!uts_ns->vdso.addr) { pr_err("Can't map vDSO pages for VE %d\n", ve->veid); for (i = 0; i < uts_ns->vdso.nr_pages; i++) put_page(new_pages[i]); kfree(new_pages); goto out_unlock; } *((int *)(uts_ns->vdso.addr + uts_ns->vdso.version_off)) = new_version; smp_wmb(); uts_ns->vdso.pages = new_pages; mutex_unlock(&vdso_mutex); pr_debug("vDSO version transition %d -> %d for VE %d\n", LINUX_VERSION_CODE, new_version, ve->veid); map_uts: return setup_additional_pages(bprm, uses_interp, uts_ns->vdso.pages, uts_ns->vdso.size); map_init_uts: return setup_additional_pages(bprm, uses_interp, init_uts_ns.vdso.pages, init_uts_ns.vdso.size); out_unlock: mutex_unlock(&vdso_mutex); return -ENOMEM; }
/** * rvt_cq_enter - add a new entry to the completion queue * @cq: completion queue * @entry: work completion entry to add * @sig: true if @entry is solicited * * This may be called with qp->s_lock held. */ void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited) { struct rvt_cq_wc *wc; unsigned long flags; u32 head; u32 next; spin_lock_irqsave(&cq->lock, flags); /* * Note that the head pointer might be writable by user processes. * Take care to verify it is a sane value. */ wc = cq->queue; head = wc->head; if (head >= (unsigned)cq->ibcq.cqe) { head = cq->ibcq.cqe; next = 0; } else { next = head + 1; } if (unlikely(next == wc->tail)) { spin_unlock_irqrestore(&cq->lock, flags); if (cq->ibcq.event_handler) { struct ib_event ev; ev.device = cq->ibcq.device; ev.element.cq = &cq->ibcq; ev.event = IB_EVENT_CQ_ERR; cq->ibcq.event_handler(&ev, cq->ibcq.cq_context); } return; } if (cq->ip) { wc->uqueue[head].wr_id = entry->wr_id; wc->uqueue[head].status = entry->status; wc->uqueue[head].opcode = entry->opcode; wc->uqueue[head].vendor_err = entry->vendor_err; wc->uqueue[head].byte_len = entry->byte_len; wc->uqueue[head].ex.imm_data = (__u32 __force)entry->ex.imm_data; wc->uqueue[head].qp_num = entry->qp->qp_num; wc->uqueue[head].src_qp = entry->src_qp; wc->uqueue[head].wc_flags = entry->wc_flags; wc->uqueue[head].pkey_index = entry->pkey_index; wc->uqueue[head].slid = entry->slid; wc->uqueue[head].sl = entry->sl; wc->uqueue[head].dlid_path_bits = entry->dlid_path_bits; wc->uqueue[head].port_num = entry->port_num; /* Make sure entry is written before the head index. */ smp_wmb(); } else { wc->kqueue[head] = *entry; } wc->head = next; if (cq->notify == IB_CQ_NEXT_COMP || (cq->notify == IB_CQ_SOLICITED && (solicited || entry->status != IB_WC_SUCCESS))) { struct kthread_worker *worker; /* * This will cause send_complete() to be called in * another thread. */ smp_read_barrier_depends(); /* see rvt_cq_exit */ worker = cq->rdi->worker; if (likely(worker)) { cq->notify = RVT_CQ_NONE; cq->triggered++; queue_kthread_work(worker, &cq->comptask); } } spin_unlock_irqrestore(&cq->lock, flags); }