void smp_call_function_interrupt(void) { void (*func) (void *info); void *info; int wait; /* call_data will be NULL if the sender timed out while * waiting on us to receive the call. */ if (!call_data) return; func = call_data->func; info = call_data->info; wait = call_data->wait; if (!wait) smp_mb__before_atomic_inc(); /* * Notify initiating CPU that I've grabbed the data and am * about to execute the function */ atomic_inc(&call_data->started); /* * At this point the info structure may be out of scope unless wait==1 */ (*func)(info); if (wait) { smp_mb__before_atomic_inc(); atomic_inc(&call_data->finished); } }
static void coupled_barrier(atomic_t *a, unsigned online) { /* * This function is effectively the same as * cpuidle_coupled_parallel_barrier, which can't be used here since * there's no cpuidle device. */ if (!coupled_coherence) return; smp_mb__before_atomic_inc(); atomic_inc(a); while (atomic_read(a) < online) cpu_relax(); if (atomic_inc_return(a) == online * 2) { atomic_set(a, 0); return; } while (atomic_read(a) > online) cpu_relax(); }
static void rcu_idle_enter_common(struct rcu_dynticks *rdtp, long long oldval) { trace_rcu_dyntick("Start", oldval, 0); if (!is_idle_task(current)) { struct task_struct *idle = idle_task(smp_processor_id()); trace_rcu_dyntick("Error on entry: not idle task", oldval, 0); ftrace_dump(DUMP_ALL); WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s", current->pid, current->comm, idle->pid, idle->comm); } rcu_prepare_for_idle(smp_processor_id()); smp_mb__before_atomic_inc(); atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); rcu_lockdep_assert(!lock_is_held(&rcu_lock_map), "Illegal idle entry in RCU read-side critical section."); rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), "Illegal idle entry in RCU-bh read-side critical section."); rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), "Illegal idle entry in RCU-sched read-side critical section."); }
void cpupri_set(struct cpupri *cp, int cpu, int newpri) { int *currpri = &cp->cpu_to_pri[cpu]; int oldpri = *currpri; int do_mb = 0; newpri = convert_prio(newpri); BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); if (newpri == oldpri) return; if (likely(newpri != CPUPRI_INVALID)) { struct cpupri_vec *vec = &cp->pri_to_cpu[newpri]; cpumask_set_cpu(cpu, vec->mask); smp_mb__before_atomic_inc(); atomic_inc(&(vec)->count); do_mb = 1; } if (likely(oldpri != CPUPRI_INVALID)) { struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; if (do_mb) smp_mb__after_atomic_inc(); atomic_dec(&(vec)->count); smp_mb__after_atomic_inc(); cpumask_clear_cpu(cpu, vec->mask); } *currpri = newpri; }
static int gpio_close(struct inode * inode, struct file * file) { smp_mb__before_atomic_inc(); atomic_inc(&gpio_open_cnt); return 0; }
void rcu_nmi_exit(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nmi_nesting == 0 || --rdtp->dynticks_nmi_nesting != 0) return; smp_mb__before_atomic_inc(); atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); }
void rcu_nmi_enter(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nmi_nesting == 0 && (atomic_read(&rdtp->dynticks) & 0x1)) return; rdtp->dynticks_nmi_nesting++; smp_mb__before_atomic_inc(); atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); }
/** * drm_update_vblank_count - update the master vblank counter * @dev: DRM device * @crtc: counter to update * * Call back into the driver to update the appropriate vblank counter * (specified by @crtc). Deal with wraparound, if it occurred, and * update the last read value so we can deal with wraparound on the next * call if necessary. * * Only necessary when going from off->on, to account for frames we * didn't get an interrupt for. * * Note: caller must hold dev->vbl_lock since this reads & writes * device vblank fields. */ static void drm_update_vblank_count(struct drm_device *dev, int crtc) { u32 cur_vblank, diff, tslot, rc; struct timeval t_vblank; /* * Interrupts were disabled prior to this call, so deal with counter * wrap if needed. * NOTE! It's possible we lost a full dev->max_vblank_count events * here if the register is small or we had vblank interrupts off for * a long time. * * We repeat the hardware vblank counter & timestamp query until * we get consistent results. This to prevent races between gpu * updating its hardware counter while we are retrieving the * corresponding vblank timestamp. */ do { cur_vblank = dev->driver->get_vblank_counter(dev, crtc); rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0); } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc)); /* Deal with counter wrap */ diff = cur_vblank - dev->last_vblank[crtc]; if (cur_vblank < dev->last_vblank[crtc]) { diff += dev->max_vblank_count; DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n", crtc, dev->last_vblank[crtc], cur_vblank, diff); } DPRINTF("enabling vblank interrupts on crtc %d, missed %d\n", crtc, diff); /* Reinitialize corresponding vblank timestamp if high-precision query * available. Skip this step if query unsupported or failed. Will * reinitialize delayed at next vblank interrupt in that case. */ if (rc) { tslot = atomic_read(&dev->_vblank_count[crtc]) + diff; vblanktimestamp(dev, crtc, tslot) = t_vblank; } smp_mb__before_atomic_inc(); atomic_add(diff, &dev->_vblank_count[crtc]); smp_mb__after_atomic_inc(); }
static void rcu_idle_exit_common(struct rcu_dynticks *rdtp, long long oldval) { smp_mb__before_atomic_inc(); atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); rcu_cleanup_after_idle(smp_processor_id()); trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting); if (!is_idle_task(current)) { struct task_struct *idle = idle_task(smp_processor_id()); trace_rcu_dyntick("Error on exit: not idle task", oldval, rdtp->dynticks_nesting); ftrace_dump(DUMP_ALL); WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s", current->pid, current->comm, idle->pid, idle->comm); } }
/** * cpupri_set - update the cpu priority setting * @cp: The cpupri context * @cpu: The target cpu * @newpri: The priority (INVALID-RT99) to assign to this CPU * * Note: Assumes cpu_rq(cpu)->lock is locked * * Returns: (void) */ void cpupri_set(struct cpupri *cp, int cpu, int newpri) { int *currpri = &cp->cpu_to_pri[cpu]; int oldpri = *currpri; int do_mb = 0; newpri = convert_prio(newpri); BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); if (newpri == oldpri) return; /* * If the cpu was currently mapped to a different value, we * need to map it to the new value then remove the old value. * Note, we must add the new value first, otherwise we risk the * cpu being missed by the priority loop in cpupri_find. */ if (likely(newpri != CPUPRI_INVALID)) { struct cpupri_vec *vec = &cp->pri_to_cpu[newpri]; cpumask_set_cpu(cpu, vec->mask); /* * When adding a new vector, we update the mask first, * do a write memory barrier, and then update the count, to * make sure the vector is visible when count is set. */ smp_mb__before_atomic_inc(); atomic_inc(&(vec)->count); do_mb = 1; } if (likely(oldpri != CPUPRI_INVALID)) { struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; /* * Because the order of modification of the vec->count * is important, we must make sure that the update * of the new prio is seen before we decrement the * old prio. This makes sure that the loop sees * one or the other when we raise the priority of * the run queue. We don't care about when we lower the * priority, as that will trigger an rt pull anyway. * * We only need to do a memory barrier if we updated * the new priority vec. */ if (do_mb) smp_mb__after_atomic_inc(); /* * When removing from the vector, we decrement the counter first * do a memory barrier and then clear the mask. */ atomic_dec(&(vec)->count); smp_mb__after_atomic_inc(); cpumask_clear_cpu(cpu, vec->mask); } *currpri = newpri; }