int __ipipe_notify_trap(int exception, struct pt_regs *regs) { struct ipipe_percpu_domain_data *p; struct ipipe_trap_data data; unsigned long flags; int ret = 0; flags = hard_local_irq_save(); /* * We send a notification about all traps raised over a * registered head domain only. */ if (__ipipe_root_p) goto out; p = ipipe_this_cpu_head_context(); if (likely(p->coflags & __IPIPE_TRAP_E)) { p->coflags |= __IPIPE_TRAP_R; hard_local_irq_restore(flags); data.exception = exception; data.regs = regs; ret = ipipe_trap_hook(&data); flags = hard_local_irq_save(); p->coflags &= ~__IPIPE_TRAP_R; } out: hard_local_irq_restore(flags); return ret; }
static void __ipipe_do_work(unsigned int virq, void *cookie) { struct ipipe_work_header *work; unsigned long flags; void *curr, *tail; int cpu; /* * Work is dispatched in enqueuing order. This interrupt * context can't migrate to another CPU. */ cpu = smp_processor_id(); curr = per_cpu(work_buf, cpu); for (;;) { flags = hard_local_irq_save(); tail = per_cpu(work_tail, cpu); if (curr == tail) { per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu); hard_local_irq_restore(flags); return; } work = curr; curr += work->size; hard_local_irq_restore(flags); work->handler(work); } }
void __ipipe_unlock_root(void) { unsigned long *p, flags; flags = hard_local_irq_save(); p = &__ipipe_root_status; __clear_bit(IPIPE_SYNCDEFER_FLAG, p); hard_local_irq_restore(flags); }
void __ipipe_complete_domain_migration(void) { unsigned long flags; ipipe_root_only(); flags = hard_local_irq_save(); complete_domain_migration(); hard_local_irq_restore(flags); }
/* * We could use standard atomic bitops in the following root status * manipulation routines, but let's prepare for SMP support in the * same move, preventing CPU migration as required. */ void __ipipe_stall_root(void) { unsigned long *p, flags; flags = hard_local_irq_save(); p = &__ipipe_root_status; __set_bit(IPIPE_STALL_FLAG, p); hard_local_irq_restore(flags); }
unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock) { unsigned long flags; int s; flags = hard_local_irq_save(); arch_spin_lock(&lock->arch_lock); s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); return arch_mangle_irq_bits(s, flags); }
unsigned long __ipipe_test_and_stall_root(void) { unsigned long *p, flags; int x; flags = hard_local_irq_save(); p = &__ipipe_root_status; x = __test_and_set_bit(IPIPE_STALL_FLAG, p); hard_local_irq_restore(flags); return x; }
int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock) { unsigned long flags; flags = hard_local_irq_save(); if (!arch_spin_trylock(&lock->arch_lock)) { hard_local_irq_restore(flags); return 0; } __set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); return 1; }
int __ipipe_notify_kevent(int kevent, void *data) { struct ipipe_percpu_domain_data *p; unsigned long flags; int ret = 0; ipipe_root_only(); flags = hard_local_irq_save(); p = ipipe_this_cpu_root_context(); if (likely(p->coflags & __IPIPE_KEVENT_E)) { p->coflags |= __IPIPE_KEVENT_R; hard_local_irq_restore(flags); ret = ipipe_kevent_hook(kevent, data); flags = hard_local_irq_save(); p->coflags &= ~__IPIPE_KEVENT_R; } hard_local_irq_restore(flags); return ret; }
int __ipipe_notify_syscall(struct pt_regs *regs) { struct ipipe_domain *caller_domain, *this_domain, *ipd; struct ipipe_percpu_domain_data *p; unsigned long flags; int ret = 0; flags = hard_local_irq_save(); caller_domain = this_domain = __ipipe_current_domain; ipd = ipipe_head_domain; next: p = ipipe_this_cpu_context(ipd); if (likely(p->coflags & __IPIPE_SYSCALL_E)) { __ipipe_set_current_context(p); p->coflags |= __IPIPE_SYSCALL_R; hard_local_irq_restore(flags); ret = ipipe_syscall_hook(caller_domain, regs); flags = hard_local_irq_save(); p->coflags &= ~__IPIPE_SYSCALL_R; if (__ipipe_current_domain != ipd) /* Account for domain migration. */ this_domain = __ipipe_current_domain; else __ipipe_set_current_domain(this_domain); } if (this_domain == ipipe_root_domain && ipd != ipipe_root_domain && ret == 0) { ipd = ipipe_root_domain; goto next; } hard_local_irq_restore(flags); return ret; }
int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock, unsigned long *x) { unsigned long flags; int s; flags = hard_local_irq_save(); if (!arch_spin_trylock(&lock->arch_lock)) { hard_local_irq_restore(flags); return 0; } s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); *x = arch_mangle_irq_bits(s, flags); return 1; }
/* * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline * just like if it has been actually received from a hw source. Also * works for virtual interrupts. */ int ipipe_trigger_irq(unsigned irq) { unsigned long flags; #ifdef CONFIG_IPIPE_DEBUG if (irq >= IPIPE_NR_IRQS || (ipipe_virtual_irq_p(irq) && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) return -EINVAL; #endif flags = hard_local_irq_save(); __ipipe_handle_irq(irq, NULL); hard_local_irq_restore(flags); return 1; }
asmlinkage void __ipipe_sync_root(void) { void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook; struct ipipe_percpu_domain_data *p; unsigned long flags; BUG_ON(irqs_disabled()); flags = hard_local_irq_save(); if (irq_tail_hook) irq_tail_hook(); clear_thread_flag(TIF_IRQ_SYNC); p = ipipe_root_cpudom_ptr(); if (__ipipe_ipending_p(p)) __ipipe_sync_pipeline(); hard_local_irq_restore(flags); }
unsigned long ipipe_critical_enter(void (*syncfn)(void)) { int cpu __maybe_unused, n __maybe_unused; unsigned long flags, loops __maybe_unused; cpumask_t allbutself __maybe_unused; flags = hard_local_irq_save(); if (num_online_cpus() == 1) return flags; #ifdef CONFIG_SMP cpu = ipipe_processor_id(); if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) { while (test_and_set_bit(0, &__ipipe_critical_lock)) { n = 0; hard_local_irq_enable(); do cpu_relax(); while (++n < cpu); hard_local_irq_disable(); } restart: spin_lock(&__ipipe_cpu_barrier); __ipipe_cpu_sync = syncfn; cpus_clear(__ipipe_cpu_pass_map); cpu_set(cpu, __ipipe_cpu_pass_map); /* * Send the sync IPI to all processors but the current * one. */ cpus_andnot(allbutself, cpu_online_map, __ipipe_cpu_pass_map); ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself); loops = IPIPE_CRITICAL_TIMEOUT; while (!cpus_equal(__ipipe_cpu_sync_map, allbutself)) { if (--loops > 0) { cpu_relax(); continue; } /* * We ran into a deadlock due to a contended * rwlock. Cancel this round and retry. */ __ipipe_cpu_sync = NULL; spin_unlock(&__ipipe_cpu_barrier); /* * Ensure all CPUs consumed the IPI to avoid * running __ipipe_cpu_sync prematurely. This * usually resolves the deadlock reason too. */ while (!cpus_equal(cpu_online_map, __ipipe_cpu_pass_map)) cpu_relax(); goto restart; } } atomic_inc(&__ipipe_critical_count); #endif /* CONFIG_SMP */ return flags; }