/* * Adjust the priority chain. Also used for deadlock detection. * Decreases task's usage by one - may thus free the task. * Returns 0 or -EDEADLK. */ static int rt_mutex_adjust_prio_chain(struct task_struct *task, int deadlock_detect, struct rt_mutex *orig_lock, struct rt_mutex_waiter *orig_waiter, struct task_struct *top_task) { struct rt_mutex *lock; struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter; int detect_deadlock, ret = 0, depth = 0; unsigned long flags; detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter, deadlock_detect); /* * The (de)boosting is a step by step approach with a lot of * pitfalls. We want this to be preemptible and we want hold a * maximum of two locks per step. So we have to check * carefully whether things change under us. */ again: if (++depth > max_lock_depth) { static int prev_max; /* * Print this only once. If the admin changes the limit, * print a new message when reaching the limit again. */ if (prev_max != max_lock_depth) { prev_max = max_lock_depth; printk(KERN_WARNING "Maximum lock depth %d reached " "task: %s (%d)\n", max_lock_depth, top_task->comm, task_pid_nr(top_task)); } put_task_struct(task); return deadlock_detect ? -EDEADLK : 0; } retry: /* * Task can not go away as we did a get_task() before ! */ raw_spin_lock_irqsave(&task->pi_lock, flags); waiter = task->pi_blocked_on; /* * Check whether the end of the boosting chain has been * reached or the state of the chain has changed while we * dropped the locks. */ if (!rt_mutex_real_waiter(waiter)) goto out_unlock_pi; /* * Check the orig_waiter state. After we dropped the locks, * the previous owner of the lock might have released the lock. */ if (orig_waiter && !rt_mutex_owner(orig_lock)) goto out_unlock_pi; /* * Drop out, when the task has no waiters. Note, * top_waiter can be NULL, when we are in the deboosting * mode! */ if (top_waiter && (!task_has_pi_waiters(task) || top_waiter != task_top_pi_waiter(task))) goto out_unlock_pi; /* * When deadlock detection is off then we check, if further * priority adjustment is necessary. */ if (!detect_deadlock && waiter->list_entry.prio == task->prio) goto out_unlock_pi; lock = waiter->lock; if (!raw_spin_trylock(&lock->wait_lock)) { raw_spin_unlock_irqrestore(&task->pi_lock, flags); cpu_relax(); goto retry; } /* Deadlock detection */ if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); raw_spin_unlock(&lock->wait_lock); ret = deadlock_detect ? -EDEADLK : 0; goto out_unlock_pi; } top_waiter = rt_mutex_top_waiter(lock); /* Requeue the waiter */ plist_del(&waiter->list_entry, &lock->wait_list); waiter->list_entry.prio = task->prio; plist_add(&waiter->list_entry, &lock->wait_list); /* Release the task */ raw_spin_unlock(&task->pi_lock); if (!rt_mutex_owner(lock)) { struct rt_mutex_waiter *lock_top_waiter; /* * If the requeue above changed the top waiter, then we need * to wake the new top waiter up to try to get the lock. */ lock_top_waiter = rt_mutex_top_waiter(lock); if (top_waiter != lock_top_waiter) { if (lock_top_waiter->savestate) wake_up_process_mutex(lock_top_waiter->task); else wake_up_process(lock_top_waiter->task); } raw_spin_unlock(&lock->wait_lock); goto out_put_task; } put_task_struct(task); /* Grab the next task */ task = rt_mutex_owner(lock); get_task_struct(task); raw_spin_lock(&task->pi_lock); if (waiter == rt_mutex_top_waiter(lock)) { /* Boost the owner */ plist_del(&top_waiter->pi_list_entry, &task->pi_waiters); waiter->pi_list_entry.prio = waiter->list_entry.prio; plist_add(&waiter->pi_list_entry, &task->pi_waiters); __rt_mutex_adjust_prio(task); } else if (top_waiter == waiter) { /* Deboost the owner */ plist_del(&waiter->pi_list_entry, &task->pi_waiters); waiter = rt_mutex_top_waiter(lock); waiter->pi_list_entry.prio = waiter->list_entry.prio; plist_add(&waiter->pi_list_entry, &task->pi_waiters); __rt_mutex_adjust_prio(task); } raw_spin_unlock(&task->pi_lock); top_waiter = rt_mutex_top_waiter(lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags); if (!detect_deadlock && waiter != top_waiter) goto out_put_task; goto again; out_unlock_pi: raw_spin_unlock_irqrestore(&task->pi_lock, flags); out_put_task: put_task_struct(task); return ret; }
/* * Try to take an rt-mutex * * Must be called with lock->wait_lock held. * * @lock: the lock to be acquired. * @task: the task which wants to acquire the lock * @waiter: the waiter that is queued to the lock's wait list. (could be NULL) */ static int do_try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, struct rt_mutex_waiter *waiter, int mode) { /* * We have to be careful here if the atomic speedups are * enabled, such that, when * - no other waiter is on the lock * - the lock has been released since we did the cmpxchg * the lock can be released or taken while we are doing the * checks and marking the lock with RT_MUTEX_HAS_WAITERS. * * The atomic acquire/release aware variant of * mark_rt_mutex_waiters uses a cmpxchg loop. After setting * the WAITERS bit, the atomic release / acquire can not * happen anymore and lock->wait_lock protects us from the * non-atomic case. * * Note, that this might set lock->owner = * RT_MUTEX_HAS_WAITERS in the case the lock is not contended * any more. This is fixed up when we take the ownership. * This is the transitional state explained at the top of this file. */ mark_rt_mutex_waiters(lock); if (rt_mutex_owner(lock)) return 0; /* * It will get the lock because of one of these conditions: * 1) there is no waiter * 2) higher priority than waiters * 3) it is top waiter */ if (rt_mutex_has_waiters(lock)) { struct task_struct *pendowner = rt_mutex_top_waiter(lock)->task; if (task != pendowner && !lock_is_stealable(task, pendowner, mode)) return 0; } /* We got the lock. */ if (waiter || rt_mutex_has_waiters(lock)) { unsigned long flags; struct rt_mutex_waiter *top; raw_spin_lock_irqsave(&task->pi_lock, flags); /* remove the queued waiter. */ if (waiter) { plist_del(&waiter->list_entry, &lock->wait_list); task->pi_blocked_on = NULL; } /* * We have to enqueue the top waiter(if it exists) into * task->pi_waiters list. */ if (rt_mutex_has_waiters(lock)) { top = rt_mutex_top_waiter(lock); top->pi_list_entry.prio = top->list_entry.prio; plist_add(&top->pi_list_entry, &task->pi_waiters); } raw_spin_unlock_irqrestore(&task->pi_lock, flags); } debug_rt_mutex_lock(lock); rt_mutex_set_owner(lock, task); rt_mutex_deadlock_account_lock(lock, task); return 1; }
void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) { s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1, spread, rq0_min_vruntime, spread0; struct rq *rq = cpu_rq(cpu); struct sched_entity *last; unsigned long flags; #ifdef CONFIG_FAIR_GROUP_SCHED SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, task_group_path(cfs_rq->tg)); #else SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu); #endif SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", SPLIT_NS(cfs_rq->exec_clock)); raw_spin_lock_irqsave(&rq->lock, flags); if (cfs_rq->rb_leftmost) MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime; last = __pick_last_entity(cfs_rq); if (last) max_vruntime = last->vruntime; min_vruntime = cfs_rq->min_vruntime; rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime; raw_spin_unlock_irqrestore(&rq->lock, flags); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime", SPLIT_NS(MIN_vruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime", SPLIT_NS(min_vruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime", SPLIT_NS(max_vruntime)); spread = max_vruntime - MIN_vruntime; SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread)); spread0 = min_vruntime - rq0_min_vruntime; SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0", SPLIT_NS(spread0)); SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over", cfs_rq->nr_spread_over); SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running); SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); #ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP SEQ_printf(m, " .%-30s: %lld\n", "runnable_load_avg", cfs_rq->runnable_load_avg); SEQ_printf(m, " .%-30s: %lld\n", "blocked_load_avg", cfs_rq->blocked_load_avg); SEQ_printf(m, " .%-30s: %lld\n", "tg_load_avg", (unsigned long long)atomic64_read(&cfs_rq->tg->load_avg)); SEQ_printf(m, " .%-30s: %lld\n", "tg_load_contrib", cfs_rq->tg_load_contrib); SEQ_printf(m, " .%-30s: %d\n", "tg_runnable_contrib", cfs_rq->tg_runnable_contrib); SEQ_printf(m, " .%-30s: %d\n", "tg->runnable_avg", atomic_read(&cfs_rq->tg->runnable_avg)); #endif #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "tg->cfs_bandwidth.timer_active", cfs_rq->tg->cfs_bandwidth.timer_active); SEQ_printf(m, " .%-30s: %d\n", "throttled", cfs_rq->throttled); SEQ_printf(m, " .%-30s: %d\n", "throttle_count", cfs_rq->throttle_count); #endif print_cfs_group_stats(m, cpu, cfs_rq->tg); #endif }
/* * Slow path lock function: */ static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, int detect_deadlock) { int ret = 0, saved_lock_depth = -1; struct rt_mutex_waiter waiter; unsigned long flags; debug_rt_mutex_init_waiter(&waiter); raw_spin_lock_irqsave(&lock->wait_lock, flags); init_lists(lock); /* Try to acquire the lock again: */ if (try_to_take_rt_mutex(lock, current, NULL)) { raw_spin_unlock_irqrestore(&lock->wait_lock, flags); return 0; } set_current_state(state); /* Setup the timer, when timeout != NULL */ if (unlikely(timeout)) { hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS); if (!hrtimer_active(&timeout->timer)) timeout->task = NULL; } ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock, flags, 0); /* * We drop the BKL here before we go into the wait loop to avoid a * possible deadlock in the scheduler. * * Note: This must be done after we call task_blocks_on_rt_mutex * because rt_release_bkl() releases the wait_lock and will * cause a race between setting the mark waiters flag in * the owner field and adding this task to the wait list. Those * two must be done within the protection of the wait_lock. */ if (unlikely(current->lock_depth >= 0)) saved_lock_depth = rt_release_bkl(lock, flags); if (likely(!ret)) ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, flags); set_current_state(TASK_RUNNING); if (unlikely(ret)) remove_waiter(lock, &waiter, flags); BUG_ON(!plist_node_empty(&waiter.list_entry)); /* * try_to_take_rt_mutex() sets the waiter bit * unconditionally. We might have to fix that up. */ fixup_rt_mutex_waiters(lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags); /* Remove pending timer: */ if (unlikely(timeout)) hrtimer_cancel(&timeout->timer); /* Must we reaquire the BKL? */ if (unlikely(saved_lock_depth >= 0)) rt_reacquire_bkl(saved_lock_depth); debug_rt_mutex_free_waiter(&waiter); return ret; }
/* Interrupts are disabled. */ void xics_migrate_irqs_away(void) { int cpu = smp_processor_id(), hw_cpu = hard_smp_processor_id(); unsigned int irq, virq; struct irq_desc *desc; /* If we used to be the default server, move to the new "boot_cpuid" */ if (hw_cpu == xics_default_server) xics_update_irq_servers(); /* Reject any interrupt that was queued to us... */ icp_ops->set_priority(0); /* Remove ourselves from the global interrupt queue */ xics_set_cpu_giq(xics_default_distrib_server, 0); /* Allow IPIs again... */ icp_ops->set_priority(DEFAULT_PRIORITY); for_each_irq_desc(virq, desc) { struct irq_chip *chip; long server; unsigned long flags; struct ics *ics; /* We can't set affinity on ISA interrupts */ if (virq < NUM_ISA_INTERRUPTS) continue; /* We only need to migrate enabled IRQS */ if (!desc->action) continue; if (desc->irq_data.domain != xics_host) continue; irq = desc->irq_data.hwirq; /* We need to get IPIs still. */ if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) continue; chip = irq_desc_get_chip(desc); if (!chip || !chip->irq_set_affinity) continue; raw_spin_lock_irqsave(&desc->lock, flags); /* Locate interrupt server */ server = -1; ics = irq_get_chip_data(virq); if (ics) server = ics->get_server(ics, irq); if (server < 0) { printk(KERN_ERR "%s: Can't find server for irq %d\n", __func__, irq); goto unlock; } /* We only support delivery to all cpus or to one cpu. * The irq has to be migrated only in the single cpu * case. */ if (server != hw_cpu) goto unlock; /* This is expected during cpu offline. */ if (cpu_online(cpu)) pr_warning("IRQ %u affinity broken off cpu %u\n", virq, cpu); /* Reset affinity to all cpus */ raw_spin_unlock_irqrestore(&desc->lock, flags); irq_set_affinity(virq, cpu_all_mask); continue; unlock: raw_spin_unlock_irqrestore(&desc->lock, flags); } }
/* The interrupt service routine for the hrtimer */ static enum hrtimer_restart rk_timer_isr(struct hrtimer *timer) { unsigned long flags; struct list_head *head; struct list_head *next_timer; cpu_tick_data_t now; cpu_tick_data_t period; rk_timer_t temp; int number_of_timers_expired = 0; int cpunum; rk_rdtsc(&now); #ifndef RK_GLOBAL_SCHED cpunum = container_of(timer, struct rk_vtimer, t)->cpunum; if (cpunum < 0 || cpunum > num_cpus) { printk("rk_timer_isr: cpunum error %d\n", cpunum); return HRTIMER_NORESTART; } if (cpunum != smp_processor_id()) { printk("rk_timer_isr: timer cpu:%d, cur cpu:%d\n", cpunum, smp_processor_id()); return HRTIMER_NORESTART; } #else cpunum = 0; #endif //printk("rk_timer_isr: enter\n"); raw_spin_lock_irqsave(ptr_rk_timer_lock(cpunum), flags); head = ptr_rk_online_timer_root(cpunum)->next; while (head != ptr_rk_online_timer_root(cpunum)) { if (number_of_timers_expired > MAX_TIMERS_PER_ISR) { printk(KERN_CRIT "BUG: We are stuck on %d\n", smp_processor_id()); goto error_spin_unlock; } temp = list_entry(head, struct rk_timer, tmr_tmr); if (temp == NULL) { printk("Null Timer Expired\n"); continue; } if (head == NULL) { printk("Head Timer is NULL\n"); continue; } next_timer = head->next; /* * Check if timer will expire within TIMER_DELTA */ if ((now + TIMER_DELTA) >= temp->tmr_expire) { number_of_timers_expired ++; // This timer has expired if (temp->overflow > var_rk_overflow(cpunum)) { var_rk_overflow(cpunum) = temp->overflow; } // Check if the next entry is valid if (head->next == NULL || head->prev == NULL || head->next == LIST_POISON1 || head->prev == LIST_POISON2) { switch (temp->tmr_type) { case TMR_JIFFY: printk("Jiffy Timer is corrupt\n"); break; case TMR_ENFORCE: printk("Enforce Timer is corrupt\n"); break; case TMR_REPLENISH_RSV: printk("Replenish Timer is corrupt %p %p\n", head->next, head->prev); break; default: printk("Unknown Timer is corrupt\n"); break; } goto error_spin_unlock; } // Delete the timer from active list rk_list_del(head); head->next = head->prev = NULL; // Unlock rk_timer_lock // - should be unlocked before calling cpu_reserve_enforce/replenish // - since the timer is already removed from the timer list, it is safe to unlock. raw_spin_unlock_irqrestore(ptr_rk_timer_lock(cpunum), flags); // Process the expired timer switch (temp->tmr_type) { case TMR_JIFFY: printk("Error: Jiffy Timer in a non-jiffy RK module?\n"); break; case TMR_ENFORCE: if (temp->reserve_link != NULL) { cpu_reserve_enforce(temp->reserve_link); } break; case TMR_REPLENISH_RSV: if (temp->reserve_link != NULL) { cpu_reserve_replenish(temp->reserve_link, &temp->tmr_expire, &period); // Make sure that we are greater than now while (temp->tmr_expire <= (now + TIMER_DELTA)) { adjust_timer(temp, period, cpunum); } rk_timer_add_cpu(temp, cpunum); } break; default: break; } } else { /* * The list of online timers is sorted * in the order of expiry times, so we can break safely here */ break; } /* * The timer list is not stable: * Enforcement timers may have been changed during the isr processing * Therefore, we need to make sure that we start from the head of the list * The assumption here is that we delete the elements as we process them * Otherwise, the next_timer may have been deleted and we cannot use it to re-enter the list * Previous code was using head = next_timer, which leads to bugs when the list is not stable */ raw_spin_lock_irqsave(ptr_rk_timer_lock(cpunum), flags); head = ptr_rk_online_timer_root(cpunum)->next; } if (!list_empty(ptr_rk_online_timer_root(cpunum))) { head = ptr_rk_online_timer_root(cpunum)->next; while (head != ptr_rk_online_timer_root(cpunum)) { temp = list_entry(head, struct rk_timer, tmr_tmr); if (temp->tmr_expire > now) { // printk("rk_timer_isr : %llu %llu\n", temp->tmr_expire, now); __rk_update_hw_timer_cpu(temp, cpunum); break; } head = head->next; } if (head == ptr_rk_online_timer_root(cpunum)){ printk("BUG: No Timers Left\n"); goto error_spin_unlock; } }
int show_interrupts(struct seq_file *p, void *v) { int i = *(loff_t *) v, j; unsigned long flags; if (i == 0) { seq_puts(p, " "); for_each_online_cpu(j) seq_printf(p, " CPU%d", j); #ifdef PARISC_IRQ_CR16_COUNTS seq_printf(p, " [min/avg/max] (CPU cycle counts)"); #endif seq_putc(p, '\n'); } if (i < NR_IRQS) { struct irq_desc *desc = irq_to_desc(i); struct irqaction *action; raw_spin_lock_irqsave(&desc->lock, flags); action = desc->action; if (!action) goto skip; seq_printf(p, "%3d: ", i); #ifdef CONFIG_SMP for_each_online_cpu(j) seq_printf(p, "%10u ", kstat_irqs_cpu(i, j)); #else seq_printf(p, "%10u ", kstat_irqs(i)); #endif seq_printf(p, " %14s", irq_desc_get_chip(desc)->name); #ifndef PARISC_IRQ_CR16_COUNTS seq_printf(p, " %s", action->name); while ((action = action->next)) seq_printf(p, ", %s", action->name); #else for ( ;action; action = action->next) { unsigned int k, avg, min, max; min = max = action->cr16_hist[0]; for (avg = k = 0; k < PARISC_CR16_HIST_SIZE; k++) { int hist = action->cr16_hist[k]; if (hist) { avg += hist; } else break; if (hist > max) max = hist; if (hist < min) min = hist; } avg /= k; seq_printf(p, " %s[%d/%d/%d]", action->name, min,avg,max); } #endif seq_putc(p, '\n'); skip: raw_spin_unlock_irqrestore(&desc->lock, flags); } return 0; }
/* * Check, if the new registered device should be used. */ static int tick_check_new_device(struct clock_event_device *newdev) { struct clock_event_device *curdev; struct tick_device *td; int cpu, ret = NOTIFY_OK; unsigned long flags; raw_spin_lock_irqsave(&tick_device_lock, flags); cpu = smp_processor_id(); if (!cpumask_test_cpu(cpu, newdev->cpumask)) goto out_bc; td = &per_cpu(tick_cpu_device, cpu); curdev = td->evtdev; /* cpu local device ? */ if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) { /* * If the cpu affinity of the device interrupt can not * be set, ignore it. */ if (!irq_can_set_affinity(newdev->irq)) goto out_bc; /* * If we have a cpu local device already, do not replace it * by a non cpu local device */ if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu))) goto out_bc; } /* * If we have an active device, then check the rating and the oneshot * feature. */ if (curdev) { /* * Prefer one shot capable devices ! */ if ((curdev->features & CLOCK_EVT_FEAT_ONESHOT) && !(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) goto out_bc; /* * Check the rating */ if (curdev->rating >= newdev->rating) goto out_bc; } /* * Replace the eventually existing device by the new * device. If the current device is the broadcast device, do * not give it back to the clockevents layer ! */ if (tick_is_broadcast_device(curdev)) { clockevents_shutdown(curdev); curdev = NULL; } clockevents_exchange_device(curdev, newdev); tick_setup_device(td, newdev, cpu, cpumask_of(cpu)); if (newdev->features & CLOCK_EVT_FEAT_ONESHOT) tick_oneshot_notify(); raw_spin_unlock_irqrestore(&tick_device_lock, flags); return NOTIFY_STOP; out_bc: /* * Can the new device be used as a broadcast device ? */ if (tick_check_broadcast_device(newdev)) ret = NOTIFY_STOP; raw_spin_unlock_irqrestore(&tick_device_lock, flags); return ret; }
/* * Powerstate information: The system enters/leaves a state, where * affected devices might stop */ static void tick_do_broadcast_on_off(unsigned long *reason) { struct clock_event_device *bc, *dev; struct tick_device *td; unsigned long flags; int cpu, bc_stopped; raw_spin_lock_irqsave(&tick_broadcast_lock, flags); cpu = smp_processor_id(); td = &per_cpu(tick_cpu_device, cpu); dev = td->evtdev; bc = tick_broadcast_device.evtdev; /* * Is the device not affected by the powerstate ? */ if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP)) goto out; if (!tick_device_is_functional(dev)) goto out; bc_stopped = cpumask_empty(tick_broadcast_mask); switch (*reason) { case CLOCK_EVT_NOTIFY_BROADCAST_ON: case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: cpumask_set_cpu(cpu, tick_broadcast_on); if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) { if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) clockevents_shutdown(dev); } if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) tick_broadcast_force = 1; break; case CLOCK_EVT_NOTIFY_BROADCAST_OFF: if (tick_broadcast_force) break; cpumask_clear_cpu(cpu, tick_broadcast_on); if (!tick_device_is_functional(dev)) break; if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) { if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) tick_setup_periodic(dev, 0); } break; } if (cpumask_empty(tick_broadcast_mask)) { if (!bc_stopped) clockevents_shutdown(bc); } else if (bc_stopped) { if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) tick_broadcast_start_periodic(bc); else tick_broadcast_setup_oneshot(bc); } out: raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); }
/** * i8259_init - Initialize the legacy controller * @node: device node of the legacy PIC (can be NULL, but then, it will match * all interrupts, so beware) * @intack_addr: PCI interrupt acknowledge (real) address which will return * the active irq from the 8259 */ void i8259_init(struct device_node *node, unsigned long intack_addr) { unsigned long flags; /* initialize the controller */ raw_spin_lock_irqsave(&i8259_lock, flags); /* Mask all first */ outb(0xff, 0xA1); outb(0xff, 0x21); /* init master interrupt controller */ outb(0x11, 0x20); /* Start init sequence */ outb(0x00, 0x21); /* Vector base */ outb(0x04, 0x21); /* edge tiggered, Cascade (slave) on IRQ2 */ outb(0x01, 0x21); /* Select 8086 mode */ /* init slave interrupt controller */ outb(0x11, 0xA0); /* Start init sequence */ outb(0x08, 0xA1); /* Vector base */ outb(0x02, 0xA1); /* edge triggered, Cascade (slave) on IRQ2 */ outb(0x01, 0xA1); /* Select 8086 mode */ /* That thing is slow */ udelay(100); /* always read ISR */ outb(0x0B, 0x20); outb(0x0B, 0xA0); /* Unmask the internal cascade */ cached_21 &= ~(1 << 2); /* Set interrupt masks */ outb(cached_A1, 0xA1); outb(cached_21, 0x21); raw_spin_unlock_irqrestore(&i8259_lock, flags); /* create a legacy host */ i8259_host = irq_alloc_host(node, IRQ_HOST_MAP_LEGACY, 0, &i8259_host_ops, 0); if (i8259_host == NULL) { printk(KERN_ERR "i8259: failed to allocate irq host !\n"); return; } /* reserve our resources */ /* XXX should we continue doing that ? it seems to cause problems * with further requesting of PCI IO resources for that range... * need to look into it. */ request_resource(&ioport_resource, &pic1_iores); request_resource(&ioport_resource, &pic2_iores); request_resource(&ioport_resource, &pic_edgectrl_iores); if (intack_addr != 0) pci_intack = ioremap(intack_addr, 1); printk(KERN_INFO "i8259 legacy interrupt controller initialized\n"); }
/* * Check, if the device is disfunctional and a place holder, which * needs to be handled by the broadcast device. */ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) { struct clock_event_device *bc = tick_broadcast_device.evtdev; unsigned long flags; int ret; raw_spin_lock_irqsave(&tick_broadcast_lock, flags); /* * Devices might be registered with both periodic and oneshot * mode disabled. This signals, that the device needs to be * operated from the broadcast device and is a placeholder for * the cpu local device. */ if (!tick_device_is_functional(dev)) { dev->event_handler = tick_handle_periodic; tick_device_setup_broadcast_func(dev); cpumask_set_cpu(cpu, tick_broadcast_mask); tick_broadcast_start_periodic(bc); ret = 1; } else { /* * Clear the broadcast bit for this cpu if the * device is not power state affected. */ if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) cpumask_clear_cpu(cpu, tick_broadcast_mask); else tick_device_setup_broadcast_func(dev); /* * Clear the broadcast bit if the CPU is not in * periodic broadcast on state. */ if (!cpumask_test_cpu(cpu, tick_broadcast_on)) cpumask_clear_cpu(cpu, tick_broadcast_mask); switch (tick_broadcast_device.mode) { case TICKDEV_MODE_ONESHOT: /* * If the system is in oneshot mode we can * unconditionally clear the oneshot mask bit, * because the CPU is running and therefore * not in an idle state which causes the power * state affected device to stop. Let the * caller initialize the device. */ tick_broadcast_clear_oneshot(cpu); ret = 0; break; case TICKDEV_MODE_PERIODIC: /* * If the system is in periodic mode, check * whether the broadcast device can be * switched off now. */ if (cpumask_empty(tick_broadcast_mask) && bc) clockevents_shutdown(bc); /* * If we kept the cpu in the broadcast mask, * tell the caller to leave the per cpu device * in shutdown state. The periodic interrupt * is delivered by the broadcast device. */ ret = cpumask_test_cpu(cpu, tick_broadcast_mask); break; default: /* Nothing to do */ ret = 0; break; } } raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); return ret; }
/* * Task blocks on lock. * * Prepare waiter and propagate pi chain * * This must be called with lock->wait_lock held. */ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, struct rt_mutex_waiter *waiter, struct task_struct *task, int detect_deadlock) { struct task_struct *owner = rt_mutex_owner(lock); struct rt_mutex_waiter *top_waiter = waiter; unsigned long flags; int chain_walk = 0, res; raw_spin_lock_irqsave(&task->pi_lock, flags); __rt_mutex_adjust_prio(task); waiter->task = task; waiter->lock = lock; plist_node_init(&waiter->list_entry, task->prio); plist_node_init(&waiter->pi_list_entry, task->prio); /* Get the top priority waiter on the lock */ if (rt_mutex_has_waiters(lock)) top_waiter = rt_mutex_top_waiter(lock); plist_add(&waiter->list_entry, &lock->wait_list); task->pi_blocked_on = waiter; raw_spin_unlock_irqrestore(&task->pi_lock, flags); if (!owner) return 0; if (waiter == rt_mutex_top_waiter(lock)) { raw_spin_lock_irqsave(&owner->pi_lock, flags); plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters); plist_add(&waiter->pi_list_entry, &owner->pi_waiters); __rt_mutex_adjust_prio(owner); if (owner->pi_blocked_on) chain_walk = 1; raw_spin_unlock_irqrestore(&owner->pi_lock, flags); } else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) chain_walk = 1; if (!chain_walk) return 0; /* * The owner can't disappear while holding a lock, * so the owner struct is protected by wait_lock. * Gets dropped in rt_mutex_adjust_prio_chain()! */ get_task_struct(owner); raw_spin_unlock(&lock->wait_lock); res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, task); raw_spin_lock(&lock->wait_lock); return res; }
/* Called from syscall or from eBPF program */ static int trie_update_elem(struct bpf_map *map, void *_key, void *value, u64 flags) { struct lpm_trie *trie = container_of(map, struct lpm_trie, map); struct lpm_trie_node *node, *im_node = NULL, *new_node = NULL; struct lpm_trie_node __rcu **slot; struct bpf_lpm_trie_key *key = _key; unsigned long irq_flags; unsigned int next_bit; size_t matchlen = 0; int ret = 0; if (unlikely(flags > BPF_EXIST)) return -EINVAL; if (key->prefixlen > trie->max_prefixlen) return -EINVAL; raw_spin_lock_irqsave(&trie->lock, irq_flags); /* Allocate and fill a new node */ if (trie->n_entries == trie->map.max_entries) { ret = -ENOSPC; goto out; } new_node = lpm_trie_node_alloc(trie, value); if (!new_node) { ret = -ENOMEM; goto out; } trie->n_entries++; new_node->prefixlen = key->prefixlen; RCU_INIT_POINTER(new_node->child[0], NULL); RCU_INIT_POINTER(new_node->child[1], NULL); memcpy(new_node->data, key->data, trie->data_size); /* Now find a slot to attach the new node. To do that, walk the tree * from the root and match as many bits as possible for each node until * we either find an empty slot or a slot that needs to be replaced by * an intermediate node. */ slot = &trie->root; while ((node = rcu_dereference_protected(*slot, lockdep_is_held(&trie->lock)))) { matchlen = longest_prefix_match(trie, node, key); if (node->prefixlen != matchlen || node->prefixlen == key->prefixlen || node->prefixlen == trie->max_prefixlen) break; next_bit = extract_bit(key->data, node->prefixlen); slot = &node->child[next_bit]; } /* If the slot is empty (a free child pointer or an empty root), * simply assign the @new_node to that slot and be done. */ if (!node) { rcu_assign_pointer(*slot, new_node); goto out; } /* If the slot we picked already exists, replace it with @new_node * which already has the correct data array set. */ if (node->prefixlen == matchlen) { new_node->child[0] = node->child[0]; new_node->child[1] = node->child[1]; if (!(node->flags & LPM_TREE_NODE_FLAG_IM)) trie->n_entries--; rcu_assign_pointer(*slot, new_node); kfree_rcu(node, rcu); goto out; } /* If the new node matches the prefix completely, it must be inserted * as an ancestor. Simply insert it between @node and *@slot. */ if (matchlen == key->prefixlen) { next_bit = extract_bit(node->data, matchlen); rcu_assign_pointer(new_node->child[next_bit], node); rcu_assign_pointer(*slot, new_node); goto out; } im_node = lpm_trie_node_alloc(trie, NULL); if (!im_node) { ret = -ENOMEM; goto out; } im_node->prefixlen = matchlen; im_node->flags |= LPM_TREE_NODE_FLAG_IM; memcpy(im_node->data, node->data, trie->data_size); /* Now determine which child to install in which slot */ if (extract_bit(key->data, matchlen)) { rcu_assign_pointer(im_node->child[0], node); rcu_assign_pointer(im_node->child[1], new_node); } else { rcu_assign_pointer(im_node->child[0], new_node); rcu_assign_pointer(im_node->child[1], node); } /* Finally, assign the intermediate node to the determined spot */ rcu_assign_pointer(*slot, im_node); out: if (ret) { if (new_node) trie->n_entries--; kfree(new_node); kfree(im_node); } raw_spin_unlock_irqrestore(&trie->lock, irq_flags); return ret; }
/* * Slow path lock function spin_lock style: this variant is very * careful not to miss any non-lock wakeups. * * The wakeup side uses wake_up_process_mutex, which, combined with * the xchg code of this function is a transparent sleep/wakeup * mechanism nested within any existing sleep/wakeup mechanism. This * enables the seemless use of arbitrary (blocking) spinlocks within * sleep/wakeup event loops. */ static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock) { struct rt_mutex_waiter waiter; unsigned long saved_state, flags; /* orig_owner is only set if next_waiter is set */ struct task_struct *uninitialized_var(orig_owner); int next_waiter; int saved_lock_depth; int ret; debug_rt_mutex_init_waiter(&waiter); waiter.task = NULL; raw_spin_lock_irqsave(&lock->wait_lock, flags); init_lists(lock); if (do_try_to_take_rt_mutex(lock, current, NULL, STEAL_LATERAL)) { raw_spin_unlock_irqrestore(&lock->wait_lock, flags); return; } BUG_ON(rt_mutex_owner(lock) == current); /* * Here we save whatever state the task was in originally, * we'll restore it at the end of the function and we'll take * any intermediate wakeup into account as well, independently * of the lock sleep/wakeup mechanism. When we get a real * wakeup the task->state is TASK_RUNNING and we change * saved_state accordingly. If we did not get a real wakeup * then we return with the saved state. We need to be careful * about original state TASK_INTERRUPTIBLE as well, as we * could miss a wakeup_interruptible() */ saved_state = rt_set_current_blocked_state(current->state); /* * Prevent schedule() to drop BKL, while waiting for * the lock ! We restore lock_depth when we come back. */ saved_lock_depth = current->lock_depth; current->lock_depth = -1; ret = task_blocks_on_rt_mutex(lock, &waiter, current, 0, flags, 1); BUG_ON(ret); for (;;) { int sleep = 1; /* Try to acquire the lock again. */ if (do_try_to_take_rt_mutex(lock, current, &waiter, STEAL_LATERAL)) break; next_waiter = &waiter == rt_mutex_top_waiter(lock); if (next_waiter) { orig_owner = rt_mutex_owner(lock); if (orig_owner) get_task_struct(orig_owner); } raw_spin_unlock_irqrestore(&lock->wait_lock, flags); debug_rt_mutex_print_deadlock(&waiter); if (next_waiter && orig_owner) { if (!adaptive_wait(&waiter, orig_owner)) sleep = 0; put_task_struct(orig_owner); } if (sleep) schedule_rt_mutex(lock); raw_spin_lock_irqsave(&lock->wait_lock, flags); saved_state = rt_set_current_blocked_state(saved_state); } current->lock_depth = saved_lock_depth; rt_restore_current_state(saved_state); /* * try_to_take_rt_mutex() sets the waiter bit * unconditionally. We might have to fix that up: */ fixup_rt_mutex_waiters(lock); BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock)); BUG_ON(!plist_node_empty(&waiter.list_entry)); raw_spin_unlock_irqrestore(&lock->wait_lock, flags); debug_rt_mutex_free_waiter(&waiter); }
void xics_migrate_irqs_away(void) { int cpu = smp_processor_id(), hw_cpu = hard_smp_processor_id(); unsigned int irq, virq; struct irq_desc *desc; if (hw_cpu == xics_default_server) xics_update_irq_servers(); icp_ops->set_priority(0); xics_set_cpu_giq(xics_default_distrib_server, 0); icp_ops->set_priority(DEFAULT_PRIORITY); for_each_irq_desc(virq, desc) { struct irq_chip *chip; long server; unsigned long flags; struct ics *ics; if (virq < NUM_ISA_INTERRUPTS) continue; if (!desc->action) continue; if (desc->irq_data.domain != xics_host) continue; irq = desc->irq_data.hwirq; if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) continue; chip = irq_desc_get_chip(desc); if (!chip || !chip->irq_set_affinity) continue; raw_spin_lock_irqsave(&desc->lock, flags); server = -1; ics = irq_get_chip_data(virq); if (ics) server = ics->get_server(ics, irq); if (server < 0) { printk(KERN_ERR "%s: Can't find server for irq %d\n", __func__, irq); goto unlock; } if (server != hw_cpu) goto unlock; if (cpu_online(cpu)) pr_warning("IRQ %u affinity broken off cpu %u\n", virq, cpu); raw_spin_unlock_irqrestore(&desc->lock, flags); irq_set_affinity(virq, cpu_all_mask); continue; unlock: raw_spin_unlock_irqrestore(&desc->lock, flags); } }
static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) { struct ir_table *table = iommu->ir_table; struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); struct irq_cfg *cfg = irq_get_chip_data(irq); u16 index, start_index; unsigned int mask = 0; unsigned long flags; int i; if (!count || !irq_iommu) return -1; /* * start the IRTE search from index 0. */ index = start_index = 0; if (count > 1) { count = __roundup_pow_of_two(count); mask = ilog2(count); } if (mask > ecap_max_handle_mask(iommu->ecap)) { printk(KERN_ERR "Requested mask %x exceeds the max invalidation handle" " mask value %Lx\n", mask, ecap_max_handle_mask(iommu->ecap)); return -1; } raw_spin_lock_irqsave(&irq_2_ir_lock, flags); do { for (i = index; i < index + count; i++) if (table->base[i].present) break; /* empty index found */ if (i == index + count) break; index = (index + count) % INTR_REMAP_TABLE_ENTRIES; if (index == start_index) { raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); printk(KERN_ERR "can't allocate an IRTE\n"); return -1; } } while (1); for (i = index; i < index + count; i++) table->base[i].present = 1; cfg->remapped = 1; irq_iommu->iommu = iommu; irq_iommu->irte_index = index; irq_iommu->sub_handle = 0; irq_iommu->irte_mask = mask; raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return index; }
static void netwinder_leds_event(led_event_t evt) { unsigned long flags; raw_spin_lock_irqsave(&leds_lock, flags); switch (evt) { case led_start: led_state |= LED_STATE_ENABLED; hw_led_state = GPIO_GREEN_LED; break; case led_stop: led_state &= ~LED_STATE_ENABLED; break; case led_claim: led_state |= LED_STATE_CLAIMED; hw_led_state = 0; break; case led_release: led_state &= ~LED_STATE_CLAIMED; hw_led_state = 0; break; #ifdef CONFIG_LEDS_TIMER case led_timer: if (!(led_state & LED_STATE_CLAIMED)) hw_led_state ^= GPIO_GREEN_LED; break; #endif #ifdef CONFIG_LEDS_CPU case led_idle_start: if (!(led_state & LED_STATE_CLAIMED)) hw_led_state &= ~GPIO_RED_LED; break; case led_idle_end: if (!(led_state & LED_STATE_CLAIMED)) hw_led_state |= GPIO_RED_LED; break; #endif case led_halted: if (!(led_state & LED_STATE_CLAIMED)) hw_led_state |= GPIO_RED_LED; break; case led_green_on: if (led_state & LED_STATE_CLAIMED) hw_led_state |= GPIO_GREEN_LED; break; case led_green_off: if (led_state & LED_STATE_CLAIMED) hw_led_state &= ~GPIO_GREEN_LED; break; case led_amber_on: if (led_state & LED_STATE_CLAIMED) hw_led_state |= GPIO_GREEN_LED | GPIO_RED_LED; break; case led_amber_off: if (led_state & LED_STATE_CLAIMED) hw_led_state &= ~(GPIO_GREEN_LED | GPIO_RED_LED); break; case led_red_on: if (led_state & LED_STATE_CLAIMED) hw_led_state |= GPIO_RED_LED; break; case led_red_off: if (led_state & LED_STATE_CLAIMED) hw_led_state &= ~GPIO_RED_LED; break; default: break; } raw_spin_unlock_irqrestore(&leds_lock, flags); if (led_state & LED_STATE_ENABLED) { raw_spin_lock_irqsave(&nw_gpio_lock, flags); nw_gpio_modify_op(GPIO_RED_LED | GPIO_GREEN_LED, hw_led_state); raw_spin_unlock_irqrestore(&nw_gpio_lock, flags); } }