/* ktap mainthread initization, main entry for ktap */ ktap_state *kp_newstate(struct ktap_parm *parm, struct dentry *dir, char **argv) { ktap_state *ks; pid_t pid; int cpu; ks = kzalloc(sizeof(ktap_state) + sizeof(ktap_global_state), GFP_KERNEL); if (!ks) return NULL; ks->stack = kp_malloc(ks, KTAP_STACK_SIZE); G(ks) = (ktap_global_state *)(ks + 1); G(ks)->mainthread = ks; G(ks)->seed = 201236; /* todo: make more random in future */ G(ks)->task = current; G(ks)->verbose = parm->verbose; /* for debug use */ G(ks)->print_timestamp = parm->print_timestamp; G(ks)->workload = parm->workload; INIT_LIST_HEAD(&(G(ks)->timers)); INIT_LIST_HEAD(&(G(ks)->probe_events_head)); G(ks)->exit = 0; if (kp_transport_init(ks, dir)) goto out; pid = (pid_t)parm->trace_pid; if (pid != -1) { struct task_struct *task; rcu_read_lock(); task = pid_task(find_vpid(pid), PIDTYPE_PID); if (!task) { kp_error(ks, "cannot find pid %d\n", pid); rcu_read_unlock(); goto out; } G(ks)->trace_task = task; get_task_struct(task); rcu_read_unlock(); } if( !alloc_cpumask_var(&G(ks)->cpumask, GFP_KERNEL)) goto out; cpumask_copy(G(ks)->cpumask, cpu_online_mask); cpu = parm->trace_cpu; if (cpu != -1) { if (!cpu_online(cpu)) { printk(KERN_INFO "ktap: cpu %d is not online\n", cpu); goto out; } cpumask_clear(G(ks)->cpumask); cpumask_set_cpu(cpu, G(ks)->cpumask); } if (cfunction_cache_init(ks)) goto out; kp_tstring_resize(ks, 512); /* set inital string hashtable size */ ktap_init_state(ks); ktap_init_registry(ks); ktap_init_arguments(ks, parm->argc, argv); /* init library */ kp_init_baselib(ks); kp_init_kdebuglib(ks); kp_init_timerlib(ks); kp_init_ansilib(ks); if (alloc_kp_percpu_data()) goto out; if (kp_probe_init(ks)) goto out; return ks; out: G(ks)->exit = 1; kp_final_exit(ks); return NULL; }
/* * This creates a new process as a copy of the old one, * but does not actually start it yet. * * It copies the registers, and all the appropriate * parts of the process environment (as per the clone * flags). The actual kick-off is left to the caller. */ static struct task_struct *copy_process(unsigned long clone_flags, unsigned long stack_start, struct pt_regs *regs, unsigned long stack_size, int __user *child_tidptr, struct pid *pid, int trace) { int retval; struct task_struct *p; int cgroup_callbacks_done = 0; if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) return ERR_PTR(-EINVAL); /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. */ if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) return ERR_PTR(-EINVAL); /* * Shared signal handlers imply shared VM. By way of the above, * thread groups also imply shared VM. Blocking this case allows * for various simplifications in other code. */ if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) return ERR_PTR(-EINVAL); /* * Siblings of global init remain as zombies on exit since they are * not reaped by their parent (swapper). To solve this and to avoid * multi-rooted process trees, prevent global and container-inits * from creating siblings. */ if ((clone_flags & CLONE_PARENT) && current->signal->flags & SIGNAL_UNKILLABLE) return ERR_PTR(-EINVAL); retval = security_task_create(clone_flags); if (retval) goto fork_out; retval = -ENOMEM; p = dup_task_struct(current); if (!p) goto fork_out; ftrace_graph_init_task(p); rt_mutex_init_task(p); #ifdef CONFIG_PROVE_LOCKING DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); #endif retval = -EAGAIN; if (atomic_read(&p->real_cred->user->processes) >= p->signal->rlim[RLIMIT_NPROC].rlim_cur) { if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) && p->real_cred->user != INIT_USER) goto bad_fork_free; } retval = copy_creds(p, clone_flags); if (retval < 0) goto bad_fork_free; /* * If multiple threads are within copy_process(), then this check * triggers too late. This doesn't hurt, the check is only there * to stop root fork bombs. */ retval = -EAGAIN; if (nr_threads >= max_threads) goto bad_fork_cleanup_count; if (!try_module_get(task_thread_info(p)->exec_domain->module)) goto bad_fork_cleanup_count; p->did_exec = 0; delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ copy_flags(clone_flags, p); INIT_LIST_HEAD(&p->children); INIT_LIST_HEAD(&p->sibling); rcu_copy_process(p); p->vfork_done = NULL; spin_lock_init(&p->alloc_lock); init_sigpending(&p->pending); p->utime = cputime_zero; p->stime = cputime_zero; p->gtime = cputime_zero; p->utimescaled = cputime_zero; p->stimescaled = cputime_zero; p->prev_utime = cputime_zero; p->prev_stime = cputime_zero; p->default_timer_slack_ns = current->timer_slack_ns; task_io_accounting_init(&p->ioac); acct_clear_integrals(p); posix_cpu_timers_init(p); p->lock_depth = -1; /* -1 = no lock */ do_posix_clock_monotonic_gettime(&p->start_time); p->real_start_time = p->start_time; monotonic_to_bootbased(&p->real_start_time); p->io_context = NULL; p->audit_context = NULL; cgroup_fork(p); #ifdef CONFIG_NUMA p->mempolicy = mpol_dup(p->mempolicy); if (IS_ERR(p->mempolicy)) { retval = PTR_ERR(p->mempolicy); p->mempolicy = NULL; goto bad_fork_cleanup_cgroup; } mpol_fix_fork_child_flag(p); #endif #ifdef CONFIG_TRACE_IRQFLAGS p->irq_events = 0; #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW p->hardirqs_enabled = 1; #else p->hardirqs_enabled = 0; #endif p->hardirq_enable_ip = 0; p->hardirq_enable_event = 0; p->hardirq_disable_ip = _THIS_IP_; p->hardirq_disable_event = 0; p->softirqs_enabled = 1; p->softirq_enable_ip = _THIS_IP_; p->softirq_enable_event = 0; p->softirq_disable_ip = 0; p->softirq_disable_event = 0; p->hardirq_context = 0; p->softirq_context = 0; #endif #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0; p->lockdep_recursion = 0; #endif #ifdef CONFIG_DEBUG_MUTEXES p->blocked_on = NULL; /* not blocked yet */ #endif p->bts = NULL; p->stack_start = stack_start; /* Perform scheduler related setup. Assign this task to a CPU. */ sched_fork(p, clone_flags); retval = perf_event_init_task(p); if (retval) goto bad_fork_cleanup_policy; if ((retval = audit_alloc(p))) goto bad_fork_cleanup_policy; /* copy all the process information */ if ((retval = copy_semundo(clone_flags, p))) goto bad_fork_cleanup_audit; if ((retval = copy_files(clone_flags, p))) goto bad_fork_cleanup_semundo; if ((retval = copy_fs(clone_flags, p))) goto bad_fork_cleanup_files; if ((retval = copy_sighand(clone_flags, p))) goto bad_fork_cleanup_fs; if ((retval = copy_signal(clone_flags, p))) goto bad_fork_cleanup_sighand; if ((retval = copy_mm(clone_flags, p))) goto bad_fork_cleanup_signal; if ((retval = copy_namespaces(clone_flags, p))) goto bad_fork_cleanup_mm; if ((retval = copy_io(clone_flags, p))) goto bad_fork_cleanup_namespaces; retval = copy_thread(clone_flags, stack_start, stack_size, p, regs); if (retval) goto bad_fork_cleanup_io; if (pid != &init_struct_pid) { retval = -ENOMEM; pid = alloc_pid(p->nsproxy->pid_ns); if (!pid) goto bad_fork_cleanup_io; if (clone_flags & CLONE_NEWPID) { retval = pid_ns_prepare_proc(p->nsproxy->pid_ns); if (retval < 0) goto bad_fork_free_pid; } } p->pid = pid_nr(pid); p->tgid = p->pid; if (clone_flags & CLONE_THREAD) p->tgid = current->tgid; if (current->nsproxy != p->nsproxy) { retval = ns_cgroup_clone(p, pid); if (retval) goto bad_fork_free_pid; } p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; /* * Clear TID on mm_release()? */ p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL; #ifdef CONFIG_FUTEX p->robust_list = NULL; #ifdef CONFIG_COMPAT p->compat_robust_list = NULL; #endif INIT_LIST_HEAD(&p->pi_state_list); p->pi_state_cache = NULL; #endif /* * sigaltstack should be cleared when sharing the same VM */ if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) p->sas_ss_sp = p->sas_ss_size = 0; /* * Syscall tracing should be turned off in the child regardless * of CLONE_PTRACE. */ clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE); #ifdef TIF_SYSCALL_EMU clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); #endif clear_all_latency_tracing(p); /* ok, now we should be set up.. */ p->exit_signal = (clone_flags & CLONE_THREAD) ? -1 : (clone_flags & CSIGNAL); p->pdeath_signal = 0; p->exit_state = 0; /* * Ok, make it visible to the rest of the system. * We dont wake it up yet. */ p->group_leader = p; INIT_LIST_HEAD(&p->thread_group); /* Now that the task is set up, run cgroup callbacks if * necessary. We need to run them before the task is visible * on the tasklist. */ cgroup_fork_callbacks(p); cgroup_callbacks_done = 1; /* Need tasklist lock for parent etc handling! */ write_lock_irq(&tasklist_lock); /* * The task hasn't been attached yet, so its cpus_allowed mask will * not be changed, nor will its assigned CPU. * * The cpus_allowed mask of the parent may have changed after it was * copied first time - so re-copy it here, then check the child's CPU * to ensure it is on a valid CPU (and if not, just force it back to * parent's CPU). This avoids alot of nasty races. */ p->cpus_allowed = current->cpus_allowed; p->rt.nr_cpus_allowed = current->rt.nr_cpus_allowed; if (unlikely(!cpu_isset(task_cpu(p), p->cpus_allowed) || !cpu_online(task_cpu(p)))) set_task_cpu(p, smp_processor_id()); /* CLONE_PARENT re-uses the old parent */ if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { p->real_parent = current->real_parent; p->parent_exec_id = current->parent_exec_id; } else { p->real_parent = current; p->parent_exec_id = current->self_exec_id; } spin_lock(¤t->sighand->siglock); /* * Process group and session signals need to be delivered to just the * parent before the fork or both the parent and the child after the * fork. Restart if a signal comes in before we add the new process to * it's process group. * A fatal signal pending means that current will exit, so the new * thread can't slip out of an OOM kill (or normal SIGKILL). */ recalc_sigpending(); if (signal_pending(current)) { spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); retval = -ERESTARTNOINTR; goto bad_fork_free_pid; } if (clone_flags & CLONE_THREAD) { atomic_inc(¤t->signal->count); atomic_inc(¤t->signal->live); p->group_leader = current->group_leader; list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group); } if (likely(p->pid)) { list_add_tail(&p->sibling, &p->real_parent->children); tracehook_finish_clone(p, clone_flags, trace); if (thread_group_leader(p)) { if (clone_flags & CLONE_NEWPID) p->nsproxy->pid_ns->child_reaper = p; p->signal->leader_pid = pid; tty_kref_put(p->signal->tty); p->signal->tty = tty_kref_get(current->signal->tty); attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); attach_pid(p, PIDTYPE_SID, task_session(current)); list_add_tail_rcu(&p->tasks, &init_task.tasks); __get_cpu_var(process_counts)++; } attach_pid(p, PIDTYPE_PID, pid); nr_threads++; } total_forks++; spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); proc_fork_connector(p); cgroup_post_fork(p); perf_event_fork(p); return p; bad_fork_free_pid: if (pid != &init_struct_pid) free_pid(pid); bad_fork_cleanup_io: put_io_context(p->io_context); bad_fork_cleanup_namespaces: exit_task_namespaces(p); bad_fork_cleanup_mm: if (p->mm) mmput(p->mm); bad_fork_cleanup_signal: if (!(clone_flags & CLONE_THREAD)) __cleanup_signal(p->signal); bad_fork_cleanup_sighand: __cleanup_sighand(p->sighand); bad_fork_cleanup_fs: exit_fs(p); /* blocking */ bad_fork_cleanup_files: exit_files(p); /* blocking */ bad_fork_cleanup_semundo: exit_sem(p); bad_fork_cleanup_audit: audit_free(p); bad_fork_cleanup_policy: perf_event_free_task(p); #ifdef CONFIG_NUMA mpol_put(p->mempolicy); bad_fork_cleanup_cgroup: #endif cgroup_exit(p, cgroup_callbacks_done); delayacct_tsk_free(p); module_put(task_thread_info(p)->exec_domain->module); bad_fork_cleanup_count: atomic_dec(&p->cred->user->processes); exit_creds(p); bad_fork_free: free_task(p); fork_out: return ERR_PTR(retval); }
/* * Can we actually use the console at this time on this cpu? * * Console drivers may assume that per-cpu resources have * been allocated. So unless they're explicitly marked as * being able to cope (CON_ANYTIME) don't call them until * this CPU is officially up. */ static inline int can_use_console(unsigned int cpu) { return cpu_online(cpu) || have_callable_console(); }
static void hotplug_decision_work_fn(struct work_struct *work) { unsigned int running, disable_load, sampling_rate, avg_running = 0; unsigned int online_cpus, available_cpus, i, j; bool hotplug_flag_on = false; bool hotplug_flag_off = false; #if DEBUG unsigned int k; #endif if (!isEnabled) return; online_cpus = num_online_cpus(); available_cpus = CPUS_AVAILABLE; disable_load = DISABLE_LOAD_THRESHOLD; // * online_cpus; //enable_load = ENABLE_LOAD_THRESHOLD; // * online_cpus; /* * Multiply nr_running() by 100 so we don't have to * use fp division to get the average. */ running = nr_running() * 100; history[index] = running; #if DEBUG pr_info("online_cpus is: %d\n", online_cpus); //pr_info("enable_load is: %d\n", enable_load); pr_info("disable_load is: %d\n", disable_load); pr_info("index is: %d\n", index); pr_info("running is: %d\n", running); #endif /* * Use a circular buffer to calculate the average load * over the sampling periods. * This will absorb load spikes of short duration where * we don't want additional cores to be onlined because * the cpufreq driver should take care of those load spikes. */ for (i = 0, j = index; i < SAMPLING_PERIODS; i++, j--) { avg_running += history[j]; if (unlikely(j == 0)) j = INDEX_MAX_VALUE; } /* * If we are at the end of the buffer, return to the beginning. */ if (unlikely(index++ == INDEX_MAX_VALUE)) index = 0; #if DEBUG pr_info("array contents: "); for (k = 0; k < SAMPLING_PERIODS; k++) { pr_info("%d: %d\t",k, history[k]); } pr_info("\n"); pr_info("avg_running before division: %d\n", avg_running); #endif avg_running = avg_running / SAMPLING_PERIODS; #if DEBUG pr_info("average_running is: %d\n", avg_running); #endif if (likely(!(flags & HOTPLUG_DISABLED))) { int cpu; for (cpu = 1; cpu < CPUS_AVAILABLE; cpu++) { if (avg_running >= enable_load[cpu] && (!cpu_online(cpu))) { hotplug_cpu_single_on[cpu] = 1; hotplug_flag_on = true; } else if (avg_running < enable_load[cpu] && (cpu_online(cpu))) { hotplug_cpu_single_off[cpu] = 1; hotplug_flag_off = true; } } if (unlikely((avg_running >= ENABLE_ALL_LOAD_THRESHOLD) && (online_cpus < available_cpus))) { pr_info("auto_hotplug: Onlining all CPUs, avg running: %d\n", avg_running); /* * Flush any delayed offlining work from the workqueue. * No point in having expensive unnecessary hotplug transitions. * We still online after flushing, because load is high enough to * warrant it. * We set the paused flag so the sampling can continue but no more * hotplug events will occur. */ flags |= HOTPLUG_PAUSED; if (delayed_work_pending(&aphotplug_offline_work)) cancel_delayed_work(&aphotplug_offline_work); hotplug_flag_on = false; schedule_work_on(0, &hotplug_online_all_work); return; } else if (flags & HOTPLUG_PAUSED) { schedule_delayed_work_on(0, &hotplug_decision_work, MIN_SAMPLING_RATE); return; } else if (hotplug_flag_on) { #if DEBUG pr_info("auto_hotplug: Onlining single CPU, avg running: %d\n", avg_running); #endif if (delayed_work_pending(&aphotplug_offline_work)) cancel_delayed_work(&aphotplug_offline_work); schedule_work_on(0, &hotplug_online_single_work); return; } else if (hotplug_flag_off) { /* Only queue a cpu_down() if there isn't one already pending */ if (!(delayed_work_pending(&aphotplug_offline_work))) { #if DEBUG pr_info("auto_hotplug: Offlining CPU, avg running: %d\n", avg_running); #endif hotplug_flag_off = false; schedule_delayed_work_on(0, &aphotplug_offline_work, HZ); } /* If boostpulse is active, clear the flags */ if (flags & BOOSTPULSE_ACTIVE) { flags &= ~BOOSTPULSE_ACTIVE; #if DEBUG pr_info("auto_hotplug: Clearing boostpulse flags\n"); #endif } } } /* * Reduce the sampling rate dynamically based on online cpus. */ sampling_rate = MIN_SAMPLING_RATE * (online_cpus * online_cpus); #if DEBUG pr_info("sampling_rate is: %d\n", jiffies_to_msecs(sampling_rate)); #endif schedule_delayed_work_on(0, &hotplug_decision_work, sampling_rate); }
/** * ixgbe_alloc_q_vector - Allocate memory for a single interrupt vector * @adapter: board private structure to initialize * @v_idx: index of vector in adapter struct * * We allocate one q_vector. If allocation fails we return -ENOMEM. **/ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, int v_idx, int txr_count, int txr_idx, int rxr_count, int rxr_idx) { struct ixgbe_q_vector *q_vector; struct ixgbe_ring *ring; int node = -1; int cpu = -1; int ring_count, size; ring_count = txr_count + rxr_count; size = sizeof(struct ixgbe_q_vector) + (sizeof(struct ixgbe_ring) * ring_count); /* customize cpu for Flow Director mapping */ if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { if (cpu_online(v_idx)) { cpu = v_idx; node = cpu_to_node(cpu); } } /* allocate q_vector and rings */ q_vector = kzalloc_node(size, GFP_KERNEL, node); if (!q_vector) q_vector = kzalloc(size, GFP_KERNEL); if (!q_vector) return -ENOMEM; /* setup affinity mask and node */ if (cpu != -1) cpumask_set_cpu(cpu, &q_vector->affinity_mask); else cpumask_copy(&q_vector->affinity_mask, cpu_online_mask); q_vector->numa_node = node; /* initialize NAPI */ netif_napi_add(adapter->netdev, &q_vector->napi, ixgbe_poll, 64); /* tie q_vector and adapter together */ adapter->q_vector[v_idx] = q_vector; q_vector->adapter = adapter; q_vector->v_idx = v_idx; /* initialize work limits */ q_vector->tx.work_limit = adapter->tx_work_limit; /* initialize pointer to rings */ ring = q_vector->ring; while (txr_count) { /* assign generic ring traits */ ring->dev = &adapter->pdev->dev; ring->netdev = adapter->netdev; /* configure backlink on ring */ ring->q_vector = q_vector; /* update q_vector Tx values */ ixgbe_add_ring(ring, &q_vector->tx); /* apply Tx specific ring traits */ ring->count = adapter->tx_ring_count; ring->queue_index = txr_idx; /* assign ring to adapter */ adapter->tx_ring[txr_idx] = ring; /* update count and index */ txr_count--; txr_idx++; /* push pointer to next ring */ ring++; } while (rxr_count) { /* assign generic ring traits */ ring->dev = &adapter->pdev->dev; ring->netdev = adapter->netdev; /* configure backlink on ring */ ring->q_vector = q_vector; /* update q_vector Rx values */ ixgbe_add_ring(ring, &q_vector->rx); /* * 82599 errata, UDP frames with a 0 checksum * can be marked as checksum errors. */ if (adapter->hw.mac.type == ixgbe_mac_82599EB) set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); #ifdef IXGBE_FCOE if (adapter->netdev->features & NETIF_F_FCOE_MTU) { struct ixgbe_ring_feature *f; f = &adapter->ring_feature[RING_F_FCOE]; if ((rxr_idx >= f->mask) && (rxr_idx < f->mask + f->indices)) set_bit(__IXGBE_RX_FCOE_BUFSZ, &ring->state); } #endif /* IXGBE_FCOE */ /* apply Rx specific ring traits */ ring->count = adapter->rx_ring_count; ring->queue_index = rxr_idx; /* assign ring to adapter */ adapter->rx_ring[rxr_idx] = ring; /* update count and index */ rxr_count--; rxr_idx++; /* push pointer to next ring */ ring++; } return 0; }
static void tick_nohz_stop_sched_tick(struct tick_sched *ts) { unsigned long seq, last_jiffies, next_jiffies, delta_jiffies; ktime_t last_update, expires, now; struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; u64 time_delta; int cpu; cpu = smp_processor_id(); ts = &per_cpu(tick_cpu_sched, cpu); now = tick_nohz_start_idle(cpu, ts); if (unlikely(!cpu_online(cpu))) { if (cpu == tick_do_timer_cpu) tick_do_timer_cpu = TICK_DO_TIMER_NONE; } if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) return; if (need_resched()) return; if (unlikely(local_softirq_pending() && cpu_online(cpu))) { static int ratelimit; if (ratelimit < 10) { printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", (unsigned int) local_softirq_pending()); ratelimit++; } return; } ts->idle_calls++; do { seq = read_seqbegin(&xtime_lock); last_update = last_jiffies_update; last_jiffies = jiffies; time_delta = timekeeping_max_deferment(); } while (read_seqretry(&xtime_lock, seq)); if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) || arch_needs_cpu(cpu)) { next_jiffies = last_jiffies + 1; delta_jiffies = 1; } else { next_jiffies = get_next_timer_interrupt(last_jiffies); delta_jiffies = next_jiffies - last_jiffies; } if (!ts->tick_stopped && delta_jiffies <= 1) goto out; if ((long)delta_jiffies >= 1) { if (cpu == tick_do_timer_cpu) { tick_do_timer_cpu = TICK_DO_TIMER_NONE; ts->do_timer_last = 1; } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) { time_delta = KTIME_MAX; ts->do_timer_last = 0; } else if (!ts->do_timer_last) { time_delta = KTIME_MAX; } if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) { time_delta = min_t(u64, time_delta, tick_period.tv64 * delta_jiffies); } if (time_delta < KTIME_MAX) expires = ktime_add_ns(last_update, time_delta); else expires.tv64 = KTIME_MAX; if (ts->tick_stopped && ktime_equal(expires, dev->next_event)) goto out; if (!ts->tick_stopped) { select_nohz_load_balancer(1); calc_load_enter_idle(); ts->idle_tick = hrtimer_get_expires(&ts->sched_timer); ts->tick_stopped = 1; ts->idle_jiffies = last_jiffies; } ts->idle_sleeps++; ts->idle_expires = expires; if (unlikely(expires.tv64 == KTIME_MAX)) { if (ts->nohz_mode == NOHZ_MODE_HIGHRES) hrtimer_cancel(&ts->sched_timer); goto out; } if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { hrtimer_start(&ts->sched_timer, expires, HRTIMER_MODE_ABS_PINNED); if (hrtimer_active(&ts->sched_timer)) goto out; } else if (!tick_program_event(expires, 0)) goto out; tick_do_update_jiffies64(ktime_get()); } raise_softirq_irqoff(TIMER_SOFTIRQ); out: ts->next_jiffies = next_jiffies; ts->last_jiffies = last_jiffies; ts->sleep_length = ktime_sub(dev->next_event, now); }
asmlinkage int vprintk(const char *fmt, va_list args) { unsigned long flags; int printed_len; char *p; static char printk_buf[1024]; static int log_level_unknown = 1; preempt_disable(); if (unlikely(oops_in_progress) && printk_cpu == smp_processor_id()) /* If a crash is occurring during printk() on this CPU, * make sure we can't deadlock */ zap_locks(); /* This stops the holder of console_sem just where we want him */ spin_lock_irqsave(&logbuf_lock, flags); printk_cpu = smp_processor_id(); /* Emit the output into the temporary buffer */ printed_len = vscnprintf(printk_buf, sizeof(printk_buf), fmt, args); #ifdef CONFIG_DEBUG_LL printascii(printk_buf); #endif /* * Copy the output into log_buf. If the caller didn't provide * appropriate log level tags, we insert them here */ for (p = printk_buf; *p; p++) { if (log_level_unknown) { /* log_level_unknown signals the start of a new line */ if (printk_time) { int loglev_char; char tbuf[50], *tp; unsigned tlen; unsigned long long t; unsigned long nanosec_rem; /* * force the log level token to be * before the time output. */ if (p[0] == '<' && p[1] >='0' && p[1] <= '7' && p[2] == '>') { loglev_char = p[1]; p += 3; printed_len += 3; } else { loglev_char = default_message_loglevel + '0'; } t = printk_clock(); nanosec_rem = do_div(t, 1000000000); tlen = sprintf(tbuf, "<%c>[%5lu.%06lu] ", loglev_char, (unsigned long)t, nanosec_rem/1000); for (tp = tbuf; tp < tbuf + tlen; tp++) emit_log_char(*tp); printed_len += tlen - 3; } else { if (p[0] != '<' || p[1] < '0' || p[1] > '7' || p[2] != '>') { emit_log_char('<'); emit_log_char(default_message_loglevel + '0'); emit_log_char('>'); } printed_len += 3; } log_level_unknown = 0; if (!*p) break; } emit_log_char(*p); if (*p == '\n') log_level_unknown = 1; } if (!cpu_online(smp_processor_id())) { /* * Some console drivers may assume that per-cpu resources have * been allocated. So don't allow them to be called by this * CPU until it is officially up. We shouldn't be calling into * random console drivers on a CPU which doesn't exist yet.. */ printk_cpu = UINT_MAX; spin_unlock_irqrestore(&logbuf_lock, flags); goto out; } if (!down_trylock(&console_sem)) { console_locked = 1; /* * We own the drivers. We can drop the spinlock and let * release_console_sem() print the text */ printk_cpu = UINT_MAX; spin_unlock_irqrestore(&logbuf_lock, flags); console_may_schedule = 0; release_console_sem(); } else { /* * Someone else owns the drivers. We drop the spinlock, which * allows the semaphore holder to proceed and to call the * console drivers with the output which we just produced. */ printk_cpu = UINT_MAX; spin_unlock_irqrestore(&logbuf_lock, flags); } out: preempt_enable(); return printed_len; }
static int show_cpuinfo(struct seq_file *m, void *v) { struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; unsigned long n = (unsigned long) v - 1; unsigned int version = cpu_data[n].processor_id; unsigned int fp_vers = cpu_data[n].fpu_id; char fmt [64]; int i; #ifdef CONFIG_SMP if (!cpu_online(n)) return 0; #endif /* * For the first processor also print the system type */ if (n == 0) { seq_printf(m, "system type\t\t: %s\n", get_system_type()); if (mips_get_machine_name()) seq_printf(m, "machine\t\t\t: %s\n", mips_get_machine_name()); } seq_printf(m, "processor\t\t: %ld\n", n); sprintf(fmt, "cpu model\t\t: %%s V%%d.%%d%s\n", cpu_data[n].options & MIPS_CPU_FPU ? " FPU V%d.%d" : ""); seq_printf(m, fmt, __cpu_name[n], (version >> 4) & 0x0f, version & 0x0f, (fp_vers >> 4) & 0x0f, fp_vers & 0x0f); seq_printf(m, "BogoMIPS\t\t: %u.%02u\n", cpu_data[n].udelay_val / (500000/HZ), (cpu_data[n].udelay_val / (5000/HZ)) % 100); seq_printf(m, "wait instruction\t: %s\n", cpu_wait ? "yes" : "no"); seq_printf(m, "microsecond timers\t: %s\n", cpu_has_counter ? "yes" : "no"); seq_printf(m, "tlb_entries\t\t: %d\n", cpu_data[n].tlbsize); seq_printf(m, "extra interrupt vector\t: %s\n", cpu_has_divec ? "yes" : "no"); seq_printf(m, "hardware watchpoint\t: %s", cpu_has_watch ? "yes, " : "no\n"); if (cpu_has_watch) { seq_printf(m, "count: %d, address/irw mask: [", cpu_data[n].watch_reg_count); for (i = 0; i < cpu_data[n].watch_reg_count; i++) seq_printf(m, "%s0x%04x", i ? ", " : "" , cpu_data[n].watch_reg_masks[i]); seq_printf(m, "]\n"); } seq_printf(m, "isa\t\t\t:"); if (cpu_has_mips_r1) seq_printf(m, " mips1"); if (cpu_has_mips_2) seq_printf(m, "%s", " mips2"); if (cpu_has_mips_3) seq_printf(m, "%s", " mips3"); if (cpu_has_mips_4) seq_printf(m, "%s", " mips4"); if (cpu_has_mips_5) seq_printf(m, "%s", " mips5"); if (cpu_has_mips32r1) seq_printf(m, "%s", " mips32r1"); if (cpu_has_mips32r2) seq_printf(m, "%s", " mips32r2"); if (cpu_has_mips32r6) seq_printf(m, "%s", " mips32r6"); if (cpu_has_mips64r1) seq_printf(m, "%s", " mips64r1"); if (cpu_has_mips64r2) seq_printf(m, "%s", " mips64r2"); if (cpu_has_mips64r6) seq_printf(m, "%s", " mips64r6"); seq_printf(m, "\n"); seq_printf(m, "ASEs implemented\t:"); if (cpu_has_mips16) seq_printf(m, "%s", " mips16"); if (cpu_has_mdmx) seq_printf(m, "%s", " mdmx"); if (cpu_has_mips3d) seq_printf(m, "%s", " mips3d"); if (cpu_has_smartmips) seq_printf(m, "%s", " smartmips"); if (cpu_has_dsp) seq_printf(m, "%s", " dsp"); if (cpu_has_dsp2) seq_printf(m, "%s", " dsp2"); if (cpu_has_mipsmt) seq_printf(m, "%s", " mt"); if (cpu_has_mmips) seq_printf(m, "%s", " micromips"); if (cpu_has_vz) seq_printf(m, "%s", " vz"); if (cpu_has_msa) seq_printf(m, "%s", " msa"); if (cpu_has_eva) seq_printf(m, "%s", " eva"); if (cpu_has_htw) seq_printf(m, "%s", " htw"); seq_printf(m, "\n"); if (cpu_has_mmips) { seq_printf(m, "micromips kernel\t: %s\n", (read_c0_config3() & MIPS_CONF3_ISA_OE) ? "yes" : "no"); } seq_printf(m, "shadow register sets\t: %d\n", cpu_data[n].srsets); seq_printf(m, "kscratch registers\t: %d\n", hweight8(cpu_data[n].kscratch_mask)); seq_printf(m, "package\t\t\t: %d\n", cpu_data[n].package); seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core); sprintf(fmt, "VCE%%c exceptions\t\t: %s\n", cpu_has_vce ? "%u" : "not available"); seq_printf(m, fmt, 'D', vced_count); seq_printf(m, fmt, 'I', vcei_count); proc_cpuinfo_notifier_args.m = m; proc_cpuinfo_notifier_args.n = n; raw_notifier_call_chain(&proc_cpuinfo_chain, 0, &proc_cpuinfo_notifier_args); seq_printf(m, "\n"); return 0; }
static void hotplug_timer(struct work_struct *work) { struct cpu_hotplug_info tmp_hotplug_info[4]; int i; unsigned int load = 0; unsigned int cpu_rq_min=0; unsigned long nr_rq_min = -1UL; unsigned int select_off_cpu = 0; enum flag flag_hotplug; mutex_lock(&hotplug_lock); if (user_lock == 1) goto no_hotplug; for_each_online_cpu(i) { struct cpu_time_info *tmp_info; cputime64_t cur_wall_time, cur_idle_time; unsigned int idle_time, wall_time; tmp_info = &per_cpu(hotplug_cpu_time, i); cur_idle_time = get_cpu_idle_time_us(i, &cur_wall_time); idle_time = (unsigned int)cputime64_sub(cur_idle_time, tmp_info->prev_cpu_idle); tmp_info->prev_cpu_idle = cur_idle_time; wall_time = (unsigned int)cputime64_sub(cur_wall_time, tmp_info->prev_cpu_wall); tmp_info->prev_cpu_wall = cur_wall_time; if (wall_time < idle_time) goto no_hotplug; tmp_info->load = 100 * (wall_time - idle_time) / wall_time; load += tmp_info->load; /*find minimum runqueue length*/ tmp_hotplug_info[i].nr_running = get_cpu_nr_running(i); if (i && nr_rq_min > tmp_hotplug_info[i].nr_running) { nr_rq_min = tmp_hotplug_info[i].nr_running; cpu_rq_min = i; } } for (i = NUM_CPUS - 1; i > 0; --i) { if (cpu_online(i) == 0) { select_off_cpu = i; break; } } /*standallone hotplug*/ flag_hotplug = standalone_hotplug(load, nr_rq_min, cpu_rq_min); /*do not ever hotplug out CPU 0*/ if ((cpu_rq_min == 0) && (flag_hotplug == HOTPLUG_OUT)) goto no_hotplug; /*cpu hotplug*/ if (flag_hotplug == HOTPLUG_IN && cpu_online(select_off_cpu) == CPU_OFF) { DBG_PRINT("cpu%d turning on!\n", select_off_cpu); cpu_up(select_off_cpu); DBG_PRINT("cpu%d on\n", select_off_cpu); hotpluging_rate = CHECK_DELAY * 4; } else if (flag_hotplug == HOTPLUG_OUT && cpu_online(cpu_rq_min) == CPU_ON) { DBG_PRINT("cpu%d turnning off!\n", cpu_rq_min); cpu_down(cpu_rq_min); DBG_PRINT("cpu%d off!\n", cpu_rq_min); hotpluging_rate = CHECK_DELAY; } no_hotplug: queue_delayed_work_on(0, hotplug_wq, &hotplug_work, hotpluging_rate); mutex_unlock(&hotplug_lock); }
int show_cpuinfo(struct seq_file *m, void *v) { int i = (int) v - 1; int err = 0; unsigned int pvr; unsigned short maj, min; unsigned long lpj; if (i >= NR_CPUS) { /* Show summary information */ #ifdef CONFIG_SMP unsigned long bogosum = 0; for (i = 0; i < NR_CPUS; ++i) if (cpu_online(i)) bogosum += cpu_data[i].loops_per_jiffy; seq_printf(m, "total bogomips\t: %lu.%02lu\n", bogosum/(500000/HZ), bogosum/(5000/HZ) % 100); #endif /* CONFIG_SMP */ if (ppc_md.show_cpuinfo != NULL) err = ppc_md.show_cpuinfo(m); return err; } #ifdef CONFIG_SMP if (!cpu_online(i)) return 0; pvr = cpu_data[i].pvr; lpj = cpu_data[i].loops_per_jiffy; #else pvr = mfspr(SPRN_PVR); lpj = loops_per_jiffy; #endif seq_printf(m, "processor\t: %d\n", i); seq_printf(m, "cpu\t\t: "); if (cur_cpu_spec->pvr_mask) seq_printf(m, "%s", cur_cpu_spec->cpu_name); else seq_printf(m, "unknown (%08x)", pvr); #ifdef CONFIG_ALTIVEC if (cur_cpu_spec->cpu_features & CPU_FTR_ALTIVEC) seq_printf(m, ", altivec supported"); #endif seq_printf(m, "\n"); #ifdef CONFIG_TAU if (cur_cpu_spec->cpu_features & CPU_FTR_TAU) { #ifdef CONFIG_TAU_AVERAGE /* more straightforward, but potentially misleading */ seq_printf(m, "temperature \t: %u C (uncalibrated)\n", cpu_temp(i)); #else /* show the actual temp sensor range */ u32 temp; temp = cpu_temp_both(i); seq_printf(m, "temperature \t: %u-%u C (uncalibrated)\n", temp & 0xff, temp >> 16); #endif }
int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int rc, c; /* * Don't allow secondary threads to come online if inhibited */ if (threads_per_core > 1 && secondaries_inhibited() && cpu % threads_per_core != 0) return -EBUSY; if (smp_ops == NULL || (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu))) return -EINVAL; cpu_idle_thread_init(cpu, tidle); /* Make sure callin-map entry is 0 (can be leftover a CPU * hotplug */ cpu_callin_map[cpu] = 0; /* The information for processor bringup must * be written out to main store before we release * the processor. */ smp_mb(); /* wake up cpus */ DBG("smp: kicking cpu %d\n", cpu); rc = smp_ops->kick_cpu(cpu); if (rc) { pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc); return rc; } /* * wait to see if the cpu made a callin (is actually up). * use this value that I found through experimentation. * -- Cort */ if (system_state < SYSTEM_RUNNING) for (c = 50000; c && !cpu_callin_map[cpu]; c--) udelay(100); #ifdef CONFIG_HOTPLUG_CPU else /* * CPUs can take much longer to come up in the * hotplug case. Wait five seconds. */ for (c = 5000; c && !cpu_callin_map[cpu]; c--) msleep(1); #endif if (!cpu_callin_map[cpu]) { printk(KERN_ERR "Processor %u is stuck.\n", cpu); return -ENOENT; } DBG("Processor %u found.\n", cpu); if (smp_ops->give_timebase) smp_ops->give_timebase(); /* Wait until cpu puts itself in the online map */ while (!cpu_online(cpu)) cpu_relax(); return 0; }
/* Requires cpu_add_remove_lock to be held */ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) { int err, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct take_cpu_down_param tcd_param = { .mod = mod, .hcpu = hcpu, }; if (num_online_cpus() == 1) return -EBUSY; if (!cpu_online(cpu)) return -EINVAL; cpu_hotplug_begin(); err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err) { nr_calls--; __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); pr_warn("%s: attempt to take down CPU %u failed\n", __func__, cpu); goto out_release; } /* * By now we've cleared cpu_active_mask, wait for all preempt-disabled * and RCU users of this state to go away such that all new such users * will observe it. * * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might * not imply sync_sched(), so explicitly call both. * * Do sync before park smpboot threads to take care the rcu boost case. */ #ifdef CONFIG_PREEMPT synchronize_sched(); #endif synchronize_rcu(); smpboot_park_threads(cpu); /* * So now all preempt/rcu users must observe !cpu_active(). */ err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); if (err) { /* CPU didn't die: tell everyone. Can't complain. */ smpboot_unpark_threads(cpu); cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); goto out_release; } BUG_ON(cpu_online(cpu)); /* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. * * Wait for the stop thread to go away. */ while (!idle_cpu(cpu)) cpu_relax(); /* This actually kills the CPU. */ __cpu_die(cpu); /* CPU is completely dead: tell everyone. Too late to complain. */ cpu_notify_nofail(CPU_DEAD | mod, hcpu); check_for_tasks(cpu); out_release: cpu_hotplug_done(); if (!err) cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); return err; }
int switch_L2(enum options option) { int i, cpu; int err = 0; int retry=0; u64 t1; u64 t2; unsigned long mask = (1<<0); if(option >= BORROW_NONE) { pr_err("wrong option %d\n", option); return -1; } t1 = sched_clock(); /* bind this process to main cpu */ while(sched_setaffinity(0, (struct cpumask*) &mask) < 0) { pr_err("Could not set cpu 0 affinity for current process(%d).\n", retry); retry++; if(retry > 100) { return -1; } } /*disable hot-plug*/ hps_set_enabled(0); is_l2_borrowed = 0; for(i=1; i<NR_CPUS; i++) { if(cpu_online(i)) { err = cpu_down(i); if(err < 0) { pr_err("[L2$ sharing] disable cpu %d failed!\n", i); hps_set_enabled(1); return -1; } } } /* disable preemption */ cpu = get_cpu(); /* enable other clusters' power */ enable_secondary_clusters_pwr(); config_L2_size(option); if(option == BORROW_L2) { is_l2_borrowed = 1; } else // if(option == RETURN_L2) { is_l2_borrowed = 0; /* Disable other clusters' power */ disable_secondary_clusters_pwr(); } /*enable hot-plug*/ hps_set_enabled(1); put_cpu(); t2 = sched_clock(); if(option == BORROW_L2) { pr_notice("[%s]: borrow L2$ cost %llu ns\n", __func__, t2 - t1); } else { pr_notice("[%s]: return L2$ cost %llu ns\n", __func__, t2 - t1); } return err; }
/* Requires cpu_add_remove_lock to be held */ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) { int mycpu, err, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct take_cpu_down_param tcd_param = { .mod = mod, .hcpu = hcpu, }; cpumask_var_t cpumask; if (num_online_cpus() == 1) return -EBUSY; if (!cpu_online(cpu)) return -EINVAL; /* Move the downtaker off the unplug cpu */ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) return -ENOMEM; cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu)); set_cpus_allowed_ptr(current, cpumask); free_cpumask_var(cpumask); migrate_disable(); mycpu = smp_processor_id(); if (mycpu == cpu) { printk(KERN_ERR "Yuck! Still on unplug CPU\n!"); migrate_enable(); return -EBUSY; } cpu_hotplug_begin(); err = cpu_unplug_begin(cpu); if (err) { printk("cpu_unplug_begin(%d) failed\n", cpu); goto out_cancel; } err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err) { nr_calls--; __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); printk("%s: attempt to take down CPU %u failed\n", __func__, cpu); goto out_release; } err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); if (err) { /* CPU didn't die: tell everyone. Can't complain. */ cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); goto out_release; } BUG_ON(cpu_online(cpu)); /* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. * * Wait for the stop thread to go away. */ while (!idle_cpu(cpu)) cpu_relax(); /* This actually kills the CPU. */ __cpu_die(cpu); /* CPU is completely dead: tell everyone. Too late to complain. */ cpu_notify_nofail(CPU_DEAD | mod, hcpu); check_for_tasks(cpu); out_release: cpu_unplug_done(cpu); out_cancel: migrate_enable(); cpu_hotplug_done(); if (!err) cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); return err; }
static int __ref __cpu_hotplug(bool out_flag, enum hotplug_cmd cmd) { int i = 0; int ret = 0; if (exynos_dm_hotplug_disabled()) return 0; #if defined(CONFIG_SCHED_HMP) if (out_flag) { if (do_disable_hotplug) goto blk_out; if (cmd == CMD_BIG_OUT && !in_low_power_mode) { for (i = setup_max_cpus - 1; i >= NR_CA7; i--) { if (cpu_online(i)) { ret = cpu_down(i); if (ret) goto blk_out; } } } else { for (i = setup_max_cpus - 1; i > 0; i--) { if (cpu_online(i)) { ret = cpu_down(i); if (ret) goto blk_out; } } } } else { if (in_suspend_prepared) goto blk_out; if (cmd == CMD_BIG_IN) { if (in_low_power_mode) goto blk_out; for (i = NR_CA7; i < setup_max_cpus; i++) { if (!cpu_online(i)) { ret = cpu_up(i); if (ret) goto blk_out; } } } else { if ((big_hotpluged && !do_disable_hotplug) || (cmd == CMD_LITTLE_IN)) { for (i = 1; i < NR_CA7; i++) { if (!cpu_online(i)) { ret = cpu_up(i); if (ret) goto blk_out; } } } else { if (lcd_is_on) { for (i = NR_CA7; i < setup_max_cpus; i++) { if (!cpu_online(i)) { if (i == NR_CA7) set_hmp_boostpulse(100000); ret = cpu_up(i); if (ret) goto blk_out; } } for (i = 1; i < NR_CA7; i++) { if (!cpu_online(i)) { ret = cpu_up(i); if (ret) goto blk_out; } } } else { for (i = 1; i < setup_max_cpus; i++) { if (!cpu_online(i)) { ret = cpu_up(i); if (ret) goto blk_out; } } } } } } #else if (out_flag) { if (do_disable_hotplug) goto blk_out; for (i = setup_max_cpus - 1; i > 0; i--) { if (cpu_online(i)) { ret = cpu_down(i); if (ret) goto blk_out; } } } else { if (in_suspend_prepared) goto blk_out; for (i = 1; i < setup_max_cpus; i++) { if (!cpu_online(i)) { ret = cpu_up(i); if (ret) goto blk_out; } } } #endif blk_out: return ret; }
static void __devinit smp_callin (void) { #ifdef XEN /* work around for spinlock irq assert. */ unsigned long flags; #endif int cpuid, phys_id; extern void ia64_init_itm(void); #ifdef CONFIG_PERFMON extern void pfm_init_percpu(void); #endif cpuid = smp_processor_id(); phys_id = hard_smp_processor_id(); if (cpu_online(cpuid)) { printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n", phys_id, cpuid); BUG(); } fix_b0_for_bsp(); #ifdef XEN notify_cpu_starting(cpuid); lock_ipi_calllock(&flags); #else lock_ipi_calllock(); #endif cpu_set(cpuid, cpu_online_map); #ifdef XEN unlock_ipi_calllock(flags); #else unlock_ipi_calllock(); #endif per_cpu(cpu_state, cpuid) = CPU_ONLINE; smp_setup_percpu_timer(); ia64_mca_cmc_vector_setup(); /* Setup vector on AP */ #ifdef CONFIG_PERFMON pfm_init_percpu(); #endif local_irq_enable(); if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) { /* * Synchronize the ITC with the BP. Need to do this after irqs are * enabled because ia64_sync_itc() calls smp_call_function_single(), which * calls spin_unlock_bh(), which calls spin_unlock_bh(), which calls * local_bh_enable(), which bugs out if irqs are not enabled... */ Dprintk("Going to syncup ITC with BP.\n"); ia64_sync_itc(0); } /* * Get our bogomips. */ ia64_init_itm(); #ifndef XEN calibrate_delay(); #endif local_cpu_data->loops_per_jiffy = loops_per_jiffy; #ifdef CONFIG_IA32_SUPPORT ia32_gdt_init(); #endif /* * Allow the master to continue. */ cpu_set(cpuid, cpu_callin_map); Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid); }
static int boost_mig_sync_thread(void *data) { int dest_cpu = (int) data; int src_cpu, ret; struct cpu_sync *s = &per_cpu(sync_info, dest_cpu); struct cpufreq_policy dest_policy; struct cpufreq_policy src_policy; unsigned long flags; while(1) { wait_event(s->sync_wq, s->pending || kthread_should_stop()); if (kthread_should_stop()) break; spin_lock_irqsave(&s->lock, flags); s->pending = false; src_cpu = s->src_cpu; spin_unlock_irqrestore(&s->lock, flags); ret = cpufreq_get_policy(&src_policy, src_cpu); if (ret) continue; ret = cpufreq_get_policy(&dest_policy, dest_cpu); if (ret) continue; if (src_policy.min == src_policy.cur && src_policy.min <= dest_policy.min) { continue; } cancel_delayed_work_sync(&s->boost_rem); if (sync_threshold) { if (src_policy.cur >= sync_threshold) s->boost_min = sync_threshold; else s->boost_min = src_policy.cur; } else { s->boost_min = src_policy.cur; } /* Force policy re-evaluation to trigger adjust notifier. */ cpufreq_update_policy(dest_cpu); /* Notify source CPU of policy change */ cpufreq_update_policy(src_cpu); #if defined(CONFIG_ARCH_MSM8974) || defined(CONFIG_ARCH_MSM8974PRO) get_online_cpus(); if (cpu_online(dest_cpu)) queue_delayed_work_on(dest_cpu, cpu_boost_wq, &s->boost_rem, msecs_to_jiffies(boost_ms)); put_online_cpus(); #else queue_delayed_work_on(s->cpu, cpu_boost_wq, &s->boost_rem, msecs_to_jiffies(boost_ms)); #endif } return 0; }
/* * This creates a new process as a copy of the old one, * but does not actually start it yet. * * It copies the registers, and all the appropriate * parts of the process environment (as per the clone * flags). The actual kick-off is left to the caller. */ static struct task_struct *copy_process(unsigned long clone_flags, unsigned long stack_start, struct pt_regs *regs, unsigned long stack_size, int __user *parent_tidptr, int __user *child_tidptr, struct pid *pid) { int retval; struct task_struct *p = NULL; if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) return ERR_PTR(-EINVAL); /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. */ if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) return ERR_PTR(-EINVAL); /* * Shared signal handlers imply shared VM. By way of the above, * thread groups also imply shared VM. Blocking this case allows * for various simplifications in other code. */ if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) return ERR_PTR(-EINVAL); retval = security_task_create(clone_flags); if (retval) goto fork_out; retval = -ENOMEM; p = dup_task_struct(current); if (!p) goto fork_out; rt_mutex_init_task(p); #ifdef CONFIG_TRACE_IRQFLAGS DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); #endif retval = -EAGAIN; if (atomic_read(&p->user->processes) >= p->signal->rlim[RLIMIT_NPROC].rlim_cur) { if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) && p->user != current->nsproxy->user_ns->root_user) goto bad_fork_free; } atomic_inc(&p->user->__count); atomic_inc(&p->user->processes); get_group_info(p->group_info); /* * If multiple threads are within copy_process(), then this check * triggers too late. This doesn't hurt, the check is only there * to stop root fork bombs. */ if (nr_threads >= max_threads) goto bad_fork_cleanup_count; if (!try_module_get(task_thread_info(p)->exec_domain->module)) goto bad_fork_cleanup_count; if (p->binfmt && !try_module_get(p->binfmt->module)) goto bad_fork_cleanup_put_domain; p->did_exec = 0; delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ copy_flags(clone_flags, p); p->pid = pid_nr(pid); retval = -EFAULT; if (clone_flags & CLONE_PARENT_SETTID) if (put_user(p->pid, parent_tidptr)) goto bad_fork_cleanup_delays_binfmt; INIT_LIST_HEAD(&p->children); INIT_LIST_HEAD(&p->sibling); p->vfork_done = NULL; spin_lock_init(&p->alloc_lock); clear_tsk_thread_flag(p, TIF_SIGPENDING); init_sigpending(&p->pending); p->utime = cputime_zero; p->stime = cputime_zero; p->prev_utime = cputime_zero; p->prev_stime = cputime_zero; #ifdef CONFIG_TASK_XACCT p->rchar = 0; /* I/O counter: bytes read */ p->wchar = 0; /* I/O counter: bytes written */ p->syscr = 0; /* I/O counter: read syscalls */ p->syscw = 0; /* I/O counter: write syscalls */ #endif task_io_accounting_init(p); acct_clear_integrals(p); p->it_virt_expires = cputime_zero; p->it_prof_expires = cputime_zero; p->it_sched_expires = 0; INIT_LIST_HEAD(&p->cpu_timers[0]); INIT_LIST_HEAD(&p->cpu_timers[1]); INIT_LIST_HEAD(&p->cpu_timers[2]); p->lock_depth = -1; /* -1 = no lock */ do_posix_clock_monotonic_gettime(&p->start_time); p->real_start_time = p->start_time; monotonic_to_bootbased(&p->real_start_time); p->security = NULL; p->io_context = NULL; p->io_wait = NULL; p->audit_context = NULL; cpuset_fork(p); #ifdef CONFIG_NUMA p->mempolicy = mpol_copy(p->mempolicy); if (IS_ERR(p->mempolicy)) { retval = PTR_ERR(p->mempolicy); p->mempolicy = NULL; goto bad_fork_cleanup_cpuset; } mpol_fix_fork_child_flag(p); #endif #ifdef CONFIG_TRACE_IRQFLAGS p->irq_events = 0; #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW p->hardirqs_enabled = 1; #else p->hardirqs_enabled = 0; #endif p->hardirq_enable_ip = 0; p->hardirq_enable_event = 0; p->hardirq_disable_ip = _THIS_IP_; p->hardirq_disable_event = 0; p->softirqs_enabled = 1; p->softirq_enable_ip = _THIS_IP_; p->softirq_enable_event = 0; p->softirq_disable_ip = 0; p->softirq_disable_event = 0; p->hardirq_context = 0; p->softirq_context = 0; #endif #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0; p->lockdep_recursion = 0; #endif #ifdef CONFIG_DEBUG_MUTEXES p->blocked_on = NULL; /* not blocked yet */ #endif p->tgid = p->pid; if (clone_flags & CLONE_THREAD) p->tgid = current->tgid; if ((retval = security_task_alloc(p))) goto bad_fork_cleanup_policy; if ((retval = audit_alloc(p))) goto bad_fork_cleanup_security; /* copy all the process information */ if ((retval = copy_semundo(clone_flags, p))) goto bad_fork_cleanup_audit; if ((retval = copy_files(clone_flags, p))) goto bad_fork_cleanup_semundo; if ((retval = copy_fs(clone_flags, p))) goto bad_fork_cleanup_files; if ((retval = copy_sighand(clone_flags, p))) goto bad_fork_cleanup_fs; if ((retval = copy_signal(clone_flags, p))) goto bad_fork_cleanup_sighand; if ((retval = copy_mm(clone_flags, p))) goto bad_fork_cleanup_signal; if ((retval = copy_keys(clone_flags, p))) goto bad_fork_cleanup_mm; if ((retval = copy_namespaces(clone_flags, p))) goto bad_fork_cleanup_keys; retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs); if (retval) goto bad_fork_cleanup_namespaces; p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; /* * Clear TID on mm_release()? */ p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL; p->robust_list = NULL; #ifdef CONFIG_COMPAT p->compat_robust_list = NULL; #endif INIT_LIST_HEAD(&p->pi_state_list); p->pi_state_cache = NULL; /* * sigaltstack should be cleared when sharing the same VM */ if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) p->sas_ss_sp = p->sas_ss_size = 0; /* * Syscall tracing should be turned off in the child regardless * of CLONE_PTRACE. */ clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE); #ifdef TIF_SYSCALL_EMU clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); #endif /* Our parent execution domain becomes current domain These must match for thread signalling to apply */ p->parent_exec_id = p->self_exec_id; /* ok, now we should be set up.. */ p->exit_signal = (clone_flags & CLONE_THREAD) ? -1 : (clone_flags & CSIGNAL); p->pdeath_signal = 0; p->exit_state = 0; /* * Ok, make it visible to the rest of the system. * We dont wake it up yet. */ p->group_leader = p; INIT_LIST_HEAD(&p->thread_group); INIT_LIST_HEAD(&p->ptrace_children); INIT_LIST_HEAD(&p->ptrace_list); /* Perform scheduler related setup. Assign this task to a CPU. */ sched_fork(p, clone_flags); /* Need tasklist lock for parent etc handling! */ write_lock_irq(&tasklist_lock); /* for sys_ioprio_set(IOPRIO_WHO_PGRP) */ p->ioprio = current->ioprio; /* * The task hasn't been attached yet, so its cpus_allowed mask will * not be changed, nor will its assigned CPU. * * The cpus_allowed mask of the parent may have changed after it was * copied first time - so re-copy it here, then check the child's CPU * to ensure it is on a valid CPU (and if not, just force it back to * parent's CPU). This avoids alot of nasty races. */ p->cpus_allowed = current->cpus_allowed; if (unlikely(!cpu_isset(task_cpu(p), p->cpus_allowed) || !cpu_online(task_cpu(p)))) set_task_cpu(p, smp_processor_id()); /* CLONE_PARENT re-uses the old parent */ if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) p->real_parent = current->real_parent; else p->real_parent = current; p->parent = p->real_parent; spin_lock(¤t->sighand->siglock); /* * Process group and session signals need to be delivered to just the * parent before the fork or both the parent and the child after the * fork. Restart if a signal comes in before we add the new process to * it's process group. * A fatal signal pending means that current will exit, so the new * thread can't slip out of an OOM kill (or normal SIGKILL). */ recalc_sigpending(); if (signal_pending(current)) { spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); retval = -ERESTARTNOINTR; goto bad_fork_cleanup_namespaces; } if (clone_flags & CLONE_THREAD) { p->group_leader = current->group_leader; list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group); if (!cputime_eq(current->signal->it_virt_expires, cputime_zero) || !cputime_eq(current->signal->it_prof_expires, cputime_zero) || current->signal->rlim[RLIMIT_CPU].rlim_cur != RLIM_INFINITY || !list_empty(¤t->signal->cpu_timers[0]) || !list_empty(¤t->signal->cpu_timers[1]) || !list_empty(¤t->signal->cpu_timers[2])) { /* * Have child wake up on its first tick to check * for process CPU timers. */ p->it_prof_expires = jiffies_to_cputime(1); } } if (likely(p->pid)) { add_parent(p); if (unlikely(p->ptrace & PT_PTRACED)) __ptrace_link(p, current->parent); if (thread_group_leader(p)) { p->signal->tty = current->signal->tty; p->signal->pgrp = process_group(current); set_signal_session(p->signal, process_session(current)); attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); attach_pid(p, PIDTYPE_SID, task_session(current)); list_add_tail_rcu(&p->tasks, &init_task.tasks); __get_cpu_var(process_counts)++; } attach_pid(p, PIDTYPE_PID, pid); nr_threads++; } total_forks++; spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); proc_fork_connector(p); return p; bad_fork_cleanup_namespaces: exit_task_namespaces(p); bad_fork_cleanup_keys: exit_keys(p); bad_fork_cleanup_mm: if (p->mm) mmput(p->mm); bad_fork_cleanup_signal: cleanup_signal(p); bad_fork_cleanup_sighand: __cleanup_sighand(p->sighand); bad_fork_cleanup_fs: exit_fs(p); /* blocking */ bad_fork_cleanup_files: exit_files(p); /* blocking */ bad_fork_cleanup_semundo: exit_sem(p); bad_fork_cleanup_audit: audit_free(p); bad_fork_cleanup_security: security_task_free(p); bad_fork_cleanup_policy: #ifdef CONFIG_NUMA mpol_free(p->mempolicy); bad_fork_cleanup_cpuset: #endif cpuset_exit(p); bad_fork_cleanup_delays_binfmt: delayacct_tsk_free(p); if (p->binfmt) module_put(p->binfmt->module); bad_fork_cleanup_put_domain: module_put(task_thread_info(p)->exec_domain->module); bad_fork_cleanup_count: put_group_info(p->group_info); atomic_dec(&p->user->processes); free_uid(p->user); bad_fork_free: free_task(p); fork_out: return ERR_PTR(retval); }
/* * Get CPU information for use by the procfs. */ static int show_cpuinfo(struct seq_file *m, void *v) { struct cpuinfo_m32r *c = v; unsigned long cpu = c - cpu_data; #ifdef CONFIG_SMP if (!cpu_online(cpu)) return 0; #endif /* CONFIG_SMP */ seq_printf(m, "processor\t: %ld\n", cpu); #if defined(CONFIG_CHIP_VDEC2) seq_printf(m, "cpu family\t: VDEC2\n" "cache size\t: Unknown\n"); #elif defined(CONFIG_CHIP_M32700) seq_printf(m,"cpu family\t: M32700\n" "cache size\t: I-8KB/D-8KB\n"); #elif defined(CONFIG_CHIP_M32102) seq_printf(m,"cpu family\t: M32102\n" "cache size\t: I-8KB\n"); #elif defined(CONFIG_CHIP_OPSP) seq_printf(m,"cpu family\t: OPSP\n" "cache size\t: I-8KB/D-8KB\n"); #elif defined(CONFIG_CHIP_MP) seq_printf(m, "cpu family\t: M32R-MP\n" "cache size\t: I-xxKB/D-xxKB\n"); #elif defined(CONFIG_CHIP_M32104) seq_printf(m,"cpu family\t: M32104\n" "cache size\t: I-8KB/D-8KB\n"); #else seq_printf(m, "cpu family\t: Unknown\n"); #endif seq_printf(m, "bogomips\t: %lu.%02lu\n", c->loops_per_jiffy/(500000/HZ), (c->loops_per_jiffy/(5000/HZ)) % 100); #if defined(CONFIG_PLAT_MAPPI) seq_printf(m, "Machine\t\t: Mappi Evaluation board\n"); #elif defined(CONFIG_PLAT_MAPPI2) seq_printf(m, "Machine\t\t: Mappi-II Evaluation board\n"); #elif defined(CONFIG_PLAT_MAPPI3) seq_printf(m, "Machine\t\t: Mappi-III Evaluation board\n"); #elif defined(CONFIG_PLAT_M32700UT) seq_printf(m, "Machine\t\t: M32700UT Evaluation board\n"); #elif defined(CONFIG_PLAT_OPSPUT) seq_printf(m, "Machine\t\t: OPSPUT Evaluation board\n"); #elif defined(CONFIG_PLAT_USRV) seq_printf(m, "Machine\t\t: uServer\n"); #elif defined(CONFIG_PLAT_OAKS32R) seq_printf(m, "Machine\t\t: OAKS32R\n"); #elif defined(CONFIG_PLAT_M32104UT) seq_printf(m, "Machine\t\t: M3T-M32104UT uT Engine board\n"); #else seq_printf(m, "Machine\t\t: Unknown\n"); #endif #define PRINT_CLOCK(name, value) \ seq_printf(m, name " clock\t: %d.%02dMHz\n", \ ((value) / 1000000), ((value) % 1000000)/10000) PRINT_CLOCK("CPU", (int)c->cpu_clock); PRINT_CLOCK("Bus", (int)c->bus_clock); seq_printf(m, "\n"); return 0; }
static int cpufreq_governor_interactivex(struct cpufreq_policy *new_policy, unsigned int event) { int rc; unsigned int min_freq = ~0; unsigned int max_freq = 0; unsigned int i; struct cpufreq_frequency_table *freq_table; switch (event) { case CPUFREQ_GOV_START: if (!cpu_online(new_policy->cpu)) return -EINVAL; /* * Do not register the idle hook and create sysfs * entries if we have already done so. */ if (atomic_inc_return(&active_count) > 1) return 0; rc = sysfs_create_group(cpufreq_global_kobject, &interactivex_attr_group); if (rc) return rc; pm_idle_old = pm_idle; pm_idle = cpufreq_idle; policy = new_policy; enabled = 1; register_early_suspend(&interactivex_power_suspend); pr_info("[imoseyon] interactiveX active\n"); freq_table = cpufreq_frequency_get_table(new_policy->cpu); for (i = 0; (freq_table[i].frequency != CPUFREQ_TABLE_END); i++) { unsigned int freq = freq_table[i].frequency; if (freq == CPUFREQ_ENTRY_INVALID) { continue; } if (freq < min_freq) min_freq = freq; if (freq > max_freq) max_freq = freq; } resum_speed = freq_table[(i-1)/2].frequency > min_freq ? freq_table[(i-1)/2].frequency : max_freq; //Value in midrange of available CPU frequencies if sufficient number of freq bins available freq_threshld = max_freq; break; case CPUFREQ_GOV_STOP: if (atomic_dec_return(&active_count) > 1) return 0; sysfs_remove_group(cpufreq_global_kobject, &interactivex_attr_group); pm_idle = pm_idle_old; del_timer(&per_cpu(cpu_timer, new_policy->cpu)); enabled = 0; unregister_early_suspend(&interactivex_power_suspend); pr_info("[imoseyon] interactiveX inactive\n"); break; case CPUFREQ_GOV_LIMITS: if (new_policy->max < new_policy->cur) __cpufreq_driver_target(new_policy, new_policy->max, CPUFREQ_RELATION_H); else if (new_policy->min > new_policy->cur) __cpufreq_driver_target(new_policy, new_policy->min, CPUFREQ_RELATION_L); break; } return 0; }
static void tick_nohz_stop_sched_tick(struct tick_sched *ts) { unsigned long seq, last_jiffies, next_jiffies, delta_jiffies; ktime_t last_update, expires, now; struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; u64 time_delta; int cpu; cpu = smp_processor_id(); ts = &per_cpu(tick_cpu_sched, cpu); now = tick_nohz_start_idle(cpu, ts); /* * If this cpu is offline and it is the one which updates * jiffies, then give up the assignment and let it be taken by * the cpu which runs the tick timer next. If we don't drop * this here the jiffies might be stale and do_timer() never * invoked. */ if (unlikely(!cpu_online(cpu))) { if (cpu == tick_do_timer_cpu) tick_do_timer_cpu = TICK_DO_TIMER_NONE; } if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) return; if (need_resched()) return; if (unlikely(local_softirq_pending() && cpu_online(cpu))) { static int ratelimit; if (ratelimit < 10) { printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", (unsigned int) local_softirq_pending()); ratelimit++; } return; } ts->idle_calls++; /* Read jiffies and the time when jiffies were updated last */ do { seq = read_seqbegin(&xtime_lock); last_update = last_jiffies_update; last_jiffies = jiffies; time_delta = timekeeping_max_deferment(); } while (read_seqretry(&xtime_lock, seq)); if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) || arch_needs_cpu(cpu)) { next_jiffies = last_jiffies + 1; delta_jiffies = 1; } else { /* Get the next timer wheel timer */ next_jiffies = get_next_timer_interrupt(last_jiffies); delta_jiffies = next_jiffies - last_jiffies; } /* * Do not stop the tick, if we are only one off * or if the cpu is required for rcu */ if (!ts->tick_stopped && delta_jiffies == 1) goto out; /* Schedule the tick, if we are at least one jiffie off */ if ((long)delta_jiffies >= 1) { /* * If this cpu is the one which updates jiffies, then * give up the assignment and let it be taken by the * cpu which runs the tick timer next, which might be * this cpu as well. If we don't drop this here the * jiffies might be stale and do_timer() never * invoked. Keep track of the fact that it was the one * which had the do_timer() duty last. If this cpu is * the one which had the do_timer() duty last, we * limit the sleep time to the timekeeping * max_deferement value which we retrieved * above. Otherwise we can sleep as long as we want. */ if (cpu == tick_do_timer_cpu) { tick_do_timer_cpu = TICK_DO_TIMER_NONE; ts->do_timer_last = 1; } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) { time_delta = KTIME_MAX; ts->do_timer_last = 0; } else if (!ts->do_timer_last) { time_delta = KTIME_MAX; } /* * calculate the expiry time for the next timer wheel * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals * that there is no timer pending or at least extremely * far into the future (12 days for HZ=1000). In this * case we set the expiry to the end of time. */ if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) { /* * Calculate the time delta for the next timer event. * If the time delta exceeds the maximum time delta * permitted by the current clocksource then adjust * the time delta accordingly to ensure the * clocksource does not wrap. */ time_delta = min_t(u64, time_delta, tick_period.tv64 * delta_jiffies); } if (time_delta < KTIME_MAX) expires = ktime_add_ns(last_update, time_delta); else expires.tv64 = KTIME_MAX; /* Skip reprogram of event if its not changed */ if (ts->tick_stopped && ktime_equal(expires, dev->next_event)) goto out; /* * nohz_stop_sched_tick can be called several times before * the nohz_restart_sched_tick is called. This happens when * interrupts arrive which do not cause a reschedule. In the * first call we save the current tick time, so we can restart * the scheduler tick in nohz_restart_sched_tick. */ if (!ts->tick_stopped) { select_nohz_load_balancer(1); ts->idle_tick = hrtimer_get_expires(&ts->sched_timer); ts->tick_stopped = 1; ts->idle_jiffies = last_jiffies; } ts->idle_sleeps++; /* Mark expires */ ts->idle_expires = expires; /* * If the expiration time == KTIME_MAX, then * in this case we simply stop the tick timer. */ if (unlikely(expires.tv64 == KTIME_MAX)) { if (ts->nohz_mode == NOHZ_MODE_HIGHRES) hrtimer_cancel(&ts->sched_timer); goto out; } if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { hrtimer_start(&ts->sched_timer, expires, HRTIMER_MODE_ABS_PINNED); /* Check, if the timer was already in the past */ if (hrtimer_active(&ts->sched_timer)) goto out; } else if (!tick_program_event(expires, 0)) goto out; /* * We are past the event already. So we crossed a * jiffie boundary. Update jiffies and raise the * softirq. */ tick_do_update_jiffies64(ktime_get()); } raise_softirq_irqoff(TIMER_SOFTIRQ); out: ts->next_jiffies = next_jiffies; ts->last_jiffies = last_jiffies; ts->sleep_length = ktime_sub(dev->next_event, now); }
/* * hps algo - hmp */ void hps_algo_hmp(void) { unsigned int cpu; unsigned int val; struct cpumask little_online_cpumask; struct cpumask big_online_cpumask; unsigned int little_num_base, little_num_limit, little_num_online; unsigned int big_num_base, big_num_limit, big_num_online; //log purpose char str1[64]; char str2[64]; int i, j; char * str1_ptr = str1; char * str2_ptr = str2; /* * run algo or not by hps_ctxt.enabled */ if (!hps_ctxt.enabled) { atomic_set(&hps_ctxt.is_ondemand, 0); return; } /* * calculate cpu loading */ hps_ctxt.cur_loads = 0; str1_ptr = str1; str2_ptr = str2; for_each_possible_cpu(cpu) { per_cpu(hps_percpu_ctxt, cpu).load = hps_cpu_get_percpu_load(cpu); hps_ctxt.cur_loads += per_cpu(hps_percpu_ctxt, cpu).load; if (hps_ctxt.cur_dump_enabled) { if (cpu_online(cpu)) i = sprintf(str1_ptr, "%4u", 1); else i = sprintf(str1_ptr, "%4u", 0); str1_ptr += i; j = sprintf(str2_ptr, "%4u", per_cpu(hps_percpu_ctxt, cpu).load); str2_ptr += j; } } hps_ctxt.cur_nr_heavy_task = hps_cpu_get_nr_heavy_task(); hps_cpu_get_tlp(&hps_ctxt.cur_tlp, &hps_ctxt.cur_iowait); /* * algo - begin */ mutex_lock(&hps_ctxt.lock); hps_ctxt.action = ACTION_NONE; atomic_set(&hps_ctxt.is_ondemand, 0); /* * algo - get boundary */ little_num_limit = min(hps_ctxt.little_num_limit_thermal, hps_ctxt.little_num_limit_low_battery); little_num_limit = min3(little_num_limit, hps_ctxt.little_num_limit_ultra_power_saving, hps_ctxt.little_num_limit_power_serv); little_num_base = hps_ctxt.little_num_base_perf_serv; cpumask_and(&little_online_cpumask, &hps_ctxt.little_cpumask, cpu_online_mask); little_num_online = cpumask_weight(&little_online_cpumask); //TODO: no need if is_hmp big_num_limit = min(hps_ctxt.big_num_limit_thermal, hps_ctxt.big_num_limit_low_battery); big_num_limit = min3(big_num_limit, hps_ctxt.big_num_limit_ultra_power_saving, hps_ctxt.big_num_limit_power_serv); big_num_base = max(hps_ctxt.cur_nr_heavy_task, hps_ctxt.big_num_base_perf_serv); cpumask_and(&big_online_cpumask, &hps_ctxt.big_cpumask, cpu_online_mask); big_num_online = cpumask_weight(&big_online_cpumask); if (hps_ctxt.cur_dump_enabled) { hps_debug(" CPU:%s\n", str1); hps_debug("LOAD:%s\n", str2); hps_debug("loads(%u), hvy_tsk(%u), tlp(%u), iowait(%u), limit_t(%u)(%u), limit_lb(%u)(%u), limit_ups(%u)(%u), limit_pos(%u)(%u), base_pes(%u)(%u)\n", hps_ctxt.cur_loads, hps_ctxt.cur_nr_heavy_task, hps_ctxt.cur_tlp, hps_ctxt.cur_iowait, hps_ctxt.little_num_limit_thermal, hps_ctxt.big_num_limit_thermal, hps_ctxt.little_num_limit_low_battery, hps_ctxt.big_num_limit_low_battery, hps_ctxt.little_num_limit_ultra_power_saving, hps_ctxt.big_num_limit_ultra_power_saving, hps_ctxt.little_num_limit_power_serv, hps_ctxt.big_num_limit_power_serv, hps_ctxt.little_num_base_perf_serv, hps_ctxt.big_num_base_perf_serv); } //ALGO_LIMIT: /* * algo - thermal, low battery */ if (big_num_online > big_num_limit) { val = big_num_online - big_num_limit; for (cpu = hps_ctxt.big_cpu_id_max; cpu >= hps_ctxt.big_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &big_online_cpumask); --big_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_LIMIT_BIG, (unsigned long *)&hps_ctxt.action); } if (little_num_online > little_num_limit) { val = little_num_online - little_num_limit; for (cpu = hps_ctxt.little_cpu_id_max; cpu > hps_ctxt.little_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &little_online_cpumask); --little_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_LIMIT_LITTLE, (unsigned long *)&hps_ctxt.action); } if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_BASE: /* * algo - PerfService, heavy task detect */ BUG_ON(big_num_online > big_num_limit); BUG_ON(little_num_online > little_num_limit); if ((big_num_online < big_num_base) && (big_num_online < big_num_limit) && (hps_ctxt.state == STATE_LATE_RESUME)) { val = min(big_num_base, big_num_limit) - big_num_online; for (cpu = hps_ctxt.big_cpu_id_min; cpu <= hps_ctxt.big_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &big_online_cpumask); ++big_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_BASE_BIG, (unsigned long *)&hps_ctxt.action); } if ((little_num_online < little_num_base) && (little_num_online < little_num_limit) && (little_num_online + big_num_online < hps_ctxt.little_num_base_perf_serv + hps_ctxt.big_num_base_perf_serv)) { val = min(little_num_base, little_num_limit) - little_num_online; if (big_num_online > hps_ctxt.big_num_base_perf_serv) val -= big_num_online - hps_ctxt.big_num_base_perf_serv; for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_BASE_LITTLE, (unsigned long *)&hps_ctxt.action); } if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; /* * update history - tlp */ val = hps_ctxt.tlp_history[hps_ctxt.tlp_history_index]; hps_ctxt.tlp_history[hps_ctxt.tlp_history_index] = hps_ctxt.cur_tlp; hps_ctxt.tlp_sum += hps_ctxt.cur_tlp; hps_ctxt.tlp_history_index = (hps_ctxt.tlp_history_index + 1 == hps_ctxt.tlp_times) ? 0 : hps_ctxt.tlp_history_index + 1; ++hps_ctxt.tlp_count; if (hps_ctxt.tlp_count > hps_ctxt.tlp_times) { BUG_ON(hps_ctxt.tlp_sum < val); hps_ctxt.tlp_sum -= val; hps_ctxt.tlp_avg = hps_ctxt.tlp_sum / hps_ctxt.tlp_times; } else { hps_ctxt.tlp_avg = hps_ctxt.tlp_sum / hps_ctxt.tlp_count; } if (hps_ctxt.stats_dump_enabled) hps_ctxt_print_algo_stats_tlp(0); //ALGO_RUSH_BOOST: /* * algo - rush boost */ if (hps_ctxt.rush_boost_enabled) { if (hps_ctxt.cur_loads > hps_ctxt.rush_boost_threshold * (little_num_online + big_num_online)) ++hps_ctxt.rush_count; else hps_ctxt.rush_count = 0; if ((hps_ctxt.rush_count >= hps_ctxt.rush_boost_times) && ((little_num_online + big_num_online) * 100 < hps_ctxt.tlp_avg)) { val = hps_ctxt.tlp_avg / 100 + (hps_ctxt.tlp_avg % 100 ? 1 : 0); BUG_ON(!(val > little_num_online + big_num_online)); if (val > num_possible_cpus()) val = num_possible_cpus(); val -= little_num_online + big_num_online; if ((val) && (little_num_online < little_num_limit)) { for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; --val; if ((val == 0) || (little_num_online == little_num_limit)) break; } } set_bit(ACTION_RUSH_BOOST_LITTLE, (unsigned long *)&hps_ctxt.action); } else if ((val) && (big_num_online < big_num_limit) && (hps_ctxt.state == STATE_LATE_RESUME)) { for (cpu = hps_ctxt.big_cpu_id_min; cpu <= hps_ctxt.big_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &big_online_cpumask); ++big_num_online; --val; if ((val == 0) || (big_num_online == big_num_limit)) break; } } set_bit(ACTION_RUSH_BOOST_BIG, (unsigned long *)&hps_ctxt.action); } } } //if (hps_ctxt.rush_boost_enabled) if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_UP: /* * algo - cpu up */ if ((little_num_online + big_num_online) < num_possible_cpus()) { /* * update history - up */ val = hps_ctxt.up_loads_history[hps_ctxt.up_loads_history_index]; hps_ctxt.up_loads_history[hps_ctxt.up_loads_history_index] = hps_ctxt.cur_loads; hps_ctxt.up_loads_sum += hps_ctxt.cur_loads; hps_ctxt.up_loads_history_index = (hps_ctxt.up_loads_history_index + 1 == hps_ctxt.up_times) ? 0 : hps_ctxt.up_loads_history_index + 1; ++hps_ctxt.up_loads_count; //XXX: use >= or >, which is benifit? use > if (hps_ctxt.up_loads_count > hps_ctxt.up_times) { BUG_ON(hps_ctxt.up_loads_sum < val); hps_ctxt.up_loads_sum -= val; } if (hps_ctxt.stats_dump_enabled) hps_ctxt_print_algo_stats_up(0); if (hps_ctxt.up_loads_count >= hps_ctxt.up_times) { if (hps_ctxt.up_loads_sum > hps_ctxt.up_threshold * hps_ctxt.up_times * (little_num_online + big_num_online)) { if (little_num_online < little_num_limit) { for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; break; } } set_bit(ACTION_UP_LITTLE, (unsigned long *)&hps_ctxt.action); } else if ((big_num_online < big_num_limit) && (hps_ctxt.state == STATE_LATE_RESUME)) { for (cpu = hps_ctxt.big_cpu_id_min; cpu <= hps_ctxt.big_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &big_online_cpumask); ++big_num_online; break; } } set_bit(ACTION_UP_BIG, (unsigned long *)&hps_ctxt.action); } } } //if (hps_ctxt.up_loads_count >= hps_ctxt.up_times) } //if ((little_num_online + big_num_online) < num_possible_cpus()) if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_DOWN: /* * algo - cpu down (inc. quick landing) */ if (little_num_online + big_num_online > 1) { /* * update history - down */ val = hps_ctxt.down_loads_history[hps_ctxt.down_loads_history_index]; hps_ctxt.down_loads_history[hps_ctxt.down_loads_history_index] = hps_ctxt.cur_loads; hps_ctxt.down_loads_sum += hps_ctxt.cur_loads; hps_ctxt.down_loads_history_index = (hps_ctxt.down_loads_history_index + 1 == hps_ctxt.down_times) ? 0 : hps_ctxt.down_loads_history_index + 1; ++hps_ctxt.down_loads_count; //XXX: use >= or >, which is benifit? use > if (hps_ctxt.down_loads_count > hps_ctxt.down_times) { BUG_ON(hps_ctxt.down_loads_sum < val); hps_ctxt.down_loads_sum -= val; } if (hps_ctxt.stats_dump_enabled) hps_ctxt_print_algo_stats_down(0); if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) { unsigned int down_threshold = hps_ctxt.down_threshold * hps_ctxt.down_times; val = little_num_online + big_num_online; while (hps_ctxt.down_loads_sum < down_threshold * (val - 1)) --val; val = little_num_online + big_num_online - val; if ((val) && (big_num_online > big_num_base)) { for (cpu = hps_ctxt.big_cpu_id_max; cpu >= hps_ctxt.big_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &big_online_cpumask); --big_num_online; --val; if ((val == 0) || (big_num_online == big_num_base)) break; } } set_bit(ACTION_DOWN_BIG, (unsigned long *)&hps_ctxt.action); } else if ((val) && (little_num_online > little_num_base)) { for (cpu = hps_ctxt.little_cpu_id_max; cpu > hps_ctxt.little_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &little_online_cpumask); --little_num_online; --val; if ((val == 0) || (little_num_online == little_num_base)) break; } } set_bit(ACTION_DOWN_LITTLE, (unsigned long *)&hps_ctxt.action); } } //if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) } //if (little_num_online + big_num_online > 1) if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_BIG_TO_LITTLE: /* * algo - b2L */ if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) { if ((little_num_online < little_num_limit) && (big_num_online > big_num_base)) { //find last online big for (val = hps_ctxt.big_cpu_id_max; val >= hps_ctxt.big_cpu_id_min; --val) { if (cpumask_test_cpu(val, &big_online_cpumask)) break; } BUG_ON(val < hps_ctxt.big_cpu_id_min); //verify whether b2L will open 1 little if (per_cpu(hps_percpu_ctxt, val).load * CPU_DMIPS_BIG_LITTLE_DIFF / 100 + hps_ctxt.up_loads_sum / hps_ctxt.up_times <= hps_ctxt.up_threshold * (little_num_online + big_num_online)) { //up 1 little for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; break; } } //down 1 big cpu_down(val); cpumask_clear_cpu(cpu, &big_online_cpumask); --big_num_online; set_bit(ACTION_BIG_TO_LITTLE, (unsigned long *)&hps_ctxt.action); } } //if ((little_num_online < little_num_limit) && (big_num_online > big_num_base)) } //if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) if (!hps_ctxt.action) goto ALGO_END_WO_ACTION; /* * algo - end */ ALGO_END_WITH_ACTION: hps_warn("(%04lx)(%u)(%u)action end(%u)(%u)(%u)(%u) (%u)(%u)(%u)(%u)(%u)(%u)(%u)(%u)(%u)(%u) (%u)(%u)(%u) (%u)(%u)(%u) (%u)(%u)(%u)(%u)(%u)\n", hps_ctxt.action, little_num_online, big_num_online, hps_ctxt.cur_loads, hps_ctxt.cur_tlp, hps_ctxt.cur_iowait, hps_ctxt.cur_nr_heavy_task, hps_ctxt.little_num_limit_thermal, hps_ctxt.big_num_limit_thermal, hps_ctxt.little_num_limit_low_battery, hps_ctxt.big_num_limit_low_battery, hps_ctxt.little_num_limit_ultra_power_saving, hps_ctxt.big_num_limit_ultra_power_saving, hps_ctxt.little_num_limit_power_serv, hps_ctxt.big_num_limit_power_serv, hps_ctxt.little_num_base_perf_serv, hps_ctxt.big_num_base_perf_serv, hps_ctxt.up_loads_sum, hps_ctxt.up_loads_count, hps_ctxt.up_loads_history_index, hps_ctxt.down_loads_sum, hps_ctxt.down_loads_count, hps_ctxt.down_loads_history_index, hps_ctxt.rush_count, hps_ctxt.tlp_sum, hps_ctxt.tlp_count, hps_ctxt.tlp_history_index, hps_ctxt.tlp_avg); hps_ctxt_reset_stas_nolock(); ALGO_END_WO_ACTION: mutex_unlock(&hps_ctxt.lock); return; }
int cpufreq_frequency_table_target(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table, unsigned int target_freq, unsigned int relation, unsigned int *index) { struct cpufreq_frequency_table optimal = { .index = ~0, .frequency = 0, }; struct cpufreq_frequency_table suboptimal = { .index = ~0, .frequency = 0, }; unsigned int i; pr_debug("request for target %u kHz (relation: %u) for cpu %u\n", target_freq, relation, policy->cpu); switch (relation) { case CPUFREQ_RELATION_H: suboptimal.frequency = ~0; break; case CPUFREQ_RELATION_L: optimal.frequency = ~0; break; } if (!cpu_online(policy->cpu)) return -EINVAL; for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { unsigned int freq = table[i].frequency; if (freq == CPUFREQ_ENTRY_INVALID) continue; if ((freq < policy->min) || (freq > policy->max)) continue; switch (relation) { case CPUFREQ_RELATION_H: if (freq <= target_freq) { if (freq >= optimal.frequency) { optimal.frequency = freq; optimal.index = i; } } else { if (freq <= suboptimal.frequency) { suboptimal.frequency = freq; suboptimal.index = i; } } break; case CPUFREQ_RELATION_L: if (freq >= target_freq) { if (freq <= optimal.frequency) { optimal.frequency = freq; optimal.index = i; } } else { if (freq >= suboptimal.frequency) { suboptimal.frequency = freq; suboptimal.index = i; } } break; } } if (optimal.index > i) { if (suboptimal.index > i) return -EINVAL; *index = suboptimal.index; } else *index = optimal.index; pr_debug("target is %u (%u kHz, %u)\n", *index, table[*index].frequency, table[*index].index); return 0; } EXPORT_SYMBOL_GPL(cpufreq_frequency_table_target); static DEFINE_PER_CPU(struct cpufreq_frequency_table *, cpufreq_show_table); static ssize_t show_available_freqs(struct cpufreq_policy *policy, char *buf) { unsigned int i = 0; unsigned int cpu = policy->cpu; ssize_t count = 0; struct cpufreq_frequency_table *table; if (!per_cpu(cpufreq_show_table, cpu)) return -ENODEV; table = per_cpu(cpufreq_show_table, cpu); for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { if (table[i].frequency == CPUFREQ_ENTRY_INVALID) continue; count += sprintf(&buf[count], "%d ", table[i].frequency); } count += sprintf(&buf[count], "\n"); return count; } struct freq_attr cpufreq_freq_attr_scaling_available_freqs = { .attr = { .name = "scaling_available_frequencies", .mode = 0444, }, .show = show_available_freqs, };
/* Requires cpu_add_remove_lock to be held */ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) { int err, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct take_cpu_down_param tcd_param = { .mod = mod, .hcpu = hcpu, }; if (num_online_cpus() == 1) return -EBUSY; if (!cpu_online(cpu)) return -EINVAL; cpu_hotplug_begin(); err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err) { nr_calls--; __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); printk("%s: attempt to take down CPU %u failed\n", __func__, cpu); goto out_release; } smpboot_park_threads(cpu); err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); if (err) { /* CPU didn't die: tell everyone. Can't complain. */ smpboot_unpark_threads(cpu); cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); goto out_release; } BUG_ON(cpu_online(cpu)); /* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. * * Wait for the stop thread to go away. */ while (!idle_cpu(cpu)) cpu_relax(); /* This actually kills the CPU. */ __cpu_die(cpu); /* CPU is completely dead: tell everyone. Too late to complain. */ cpu_notify_nofail(CPU_DEAD | mod, hcpu); check_for_tasks(cpu); out_release: cpu_hotplug_done(); if (!err) cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); return err; }
static int boost_mig_sync_thread(void *data) { int dest_cpu = (int) data; int src_cpu, ret; struct cpu_sync *s = &per_cpu(sync_info, dest_cpu); struct cpufreq_policy dest_policy; struct cpufreq_policy src_policy; unsigned long flags; unsigned int req_freq; if (!cpuboost_enable) return 0; while (1) { wait_event(s->sync_wq, s->pending || kthread_should_stop()); #ifdef CONFIG_IRLED_GPIO if (unlikely(gir_boost_disable)) { pr_debug("[GPIO_IR][%s] continue~!(cpu:%d)\n", __func__, raw_smp_processor_id()); continue; } #endif if (kthread_should_stop()) break; spin_lock_irqsave(&s->lock, flags); s->pending = false; src_cpu = s->src_cpu; spin_unlock_irqrestore(&s->lock, flags); ret = cpufreq_get_policy(&src_policy, src_cpu); if (ret) continue; ret = cpufreq_get_policy(&dest_policy, dest_cpu); if (ret) continue; if (s->task_load < migration_load_threshold) continue; req_freq = load_based_syncs ? (dest_policy.max * s->task_load) / 100 : src_policy.cur; if (req_freq <= dest_policy.cpuinfo.min_freq) { pr_debug("No sync. Sync Freq:%u\n", req_freq); continue; } if (sync_threshold) req_freq = min(sync_threshold, req_freq); cancel_delayed_work_sync(&s->boost_rem); s->boost_min = req_freq; /* Force policy re-evaluation to trigger adjust notifier. */ get_online_cpus(); if (cpu_online(src_cpu)) /* * Send an unchanged policy update to the source * CPU. Even though the policy isn't changed from * its existing boosted or non-boosted state * notifying the source CPU will let the governor * know a boost happened on another CPU and that it * should re-evaluate the frequency at the next timer * event without interference from a min sample time. */ cpufreq_update_policy(src_cpu); if (cpu_online(dest_cpu)) { cpufreq_update_policy(dest_cpu); queue_delayed_work_on(dest_cpu, cpu_boost_wq, &s->boost_rem, msecs_to_jiffies(boost_ms)); } else { s->boost_min = 0; } put_online_cpus(); } return 0; }
asmlinkage int vprintk(const char *fmt, va_list args) { unsigned long flags; int printed_len; char *p; static char printk_buf[1024]; static int log_level_unknown = 1; preempt_disable(); if (unlikely(oops_in_progress) && printk_cpu == smp_processor_id()) /* If a crash is occurring during printk() on this CPU, * make sure we can't deadlock */ zap_locks(); /* This stops the holder of console_sem just where we want him */ raw_local_irq_save(flags); lockdep_off(); spin_lock(&logbuf_lock); printk_cpu = smp_processor_id(); /* Emit the output into the temporary buffer */ printed_len = vscnprintf(printk_buf, sizeof(printk_buf), fmt, args); /* * Copy the output into log_buf. If the caller didn't provide * appropriate log level tags, we insert them here */ for (p = printk_buf; *p; p++) { if (log_level_unknown) { /* log_level_unknown signals the start of a new line */ if (printk_time) { int loglev_char; char tbuf[50], *tp; unsigned tlen; unsigned long long t; unsigned long nanosec_rem; /* * force the log level token to be * before the time output. */ if (p[0] == '<' && p[1] >='0' && p[1] <= '7' && p[2] == '>') { loglev_char = p[1]; p += 3; printed_len -= 3; } else { loglev_char = default_message_loglevel + '0'; } t = printk_clock(); nanosec_rem = do_div(t, 1000000000); tlen = sprintf(tbuf, "<%c>[%5lu.%06lu] ", loglev_char, (unsigned long)t, nanosec_rem/1000); for (tp = tbuf; tp < tbuf + tlen; tp++) emit_log_char(*tp); printed_len += tlen; } else { if (p[0] != '<' || p[1] < '0' || p[1] > '7' || p[2] != '>') { emit_log_char('<'); emit_log_char(default_message_loglevel + '0'); emit_log_char('>'); printed_len += 3; } } log_level_unknown = 0; if (!*p) break; } emit_log_char(*p); if (*p == '\n') log_level_unknown = 1; } if (!down_trylock(&console_sem)) { /* * We own the drivers. We can drop the spinlock and * let release_console_sem() print the text, maybe ... */ console_locked = 1; printk_cpu = UINT_MAX; spin_unlock(&logbuf_lock); /* * Console drivers may assume that per-cpu resources have * been allocated. So unless they're explicitly marked as * being able to cope (CON_ANYTIME) don't call them until * this CPU is officially up. */ if (cpu_online(smp_processor_id()) || have_callable_console()) { console_may_schedule = 0; release_console_sem(); } else { /* Release by hand to avoid flushing the buffer. */ console_locked = 0; up(&console_sem); } lockdep_on(); raw_local_irq_restore(flags); } else { /* * Someone else owns the drivers. We drop the spinlock, which * allows the semaphore holder to proceed and to call the * console drivers with the output which we just produced. */ printk_cpu = UINT_MAX; spin_unlock(&logbuf_lock); lockdep_on(); raw_local_irq_restore(flags); } preempt_enable(); return printed_len; }
void xics_migrate_irqs_away(void) { int cpu = smp_processor_id(), hw_cpu = hard_smp_processor_id(); unsigned int irq, virq; struct irq_desc *desc; if (hw_cpu == xics_default_server) xics_update_irq_servers(); icp_ops->set_priority(0); xics_set_cpu_giq(xics_default_distrib_server, 0); icp_ops->set_priority(DEFAULT_PRIORITY); for_each_irq_desc(virq, desc) { struct irq_chip *chip; long server; unsigned long flags; struct ics *ics; if (virq < NUM_ISA_INTERRUPTS) continue; if (!desc->action) continue; if (desc->irq_data.domain != xics_host) continue; irq = desc->irq_data.hwirq; if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) continue; chip = irq_desc_get_chip(desc); if (!chip || !chip->irq_set_affinity) continue; raw_spin_lock_irqsave(&desc->lock, flags); server = -1; ics = irq_get_chip_data(virq); if (ics) server = ics->get_server(ics, irq); if (server < 0) { printk(KERN_ERR "%s: Can't find server for irq %d\n", __func__, irq); goto unlock; } if (server != hw_cpu) goto unlock; if (cpu_online(cpu)) pr_warning("IRQ %u affinity broken off cpu %u\n", virq, cpu); raw_spin_unlock_irqrestore(&desc->lock, flags); irq_set_affinity(virq, cpu_all_mask); continue; unlock: raw_spin_unlock_irqrestore(&desc->lock, flags); } }
int __devinit __cpu_up(unsigned int cpu) { int c; secondary_ti = current_set[cpu]; if (!cpu_enable(cpu)) return 0; if (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)) return -EINVAL; /* Make sure callin-map entry is 0 (can be leftover a CPU * hotplug */ cpu_callin_map[cpu] = 0; /* The information for processor bringup must * be written out to main store before we release * the processor. */ smp_mb(); /* wake up cpus */ DBG("smp: kicking cpu %d\n", cpu); smp_ops->kick_cpu(cpu); /* * wait to see if the cpu made a callin (is actually up). * use this value that I found through experimentation. * -- Cort */ if (system_state < SYSTEM_RUNNING) for (c = 5000; c && !cpu_callin_map[cpu]; c--) udelay(100); #ifdef CONFIG_HOTPLUG_CPU else /* * CPUs can take much longer to come up in the * hotplug case. Wait five seconds. */ for (c = 25; c && !cpu_callin_map[cpu]; c--) { msleep(200); } #endif if (!cpu_callin_map[cpu]) { printk("Processor %u is stuck.\n", cpu); return -ENOENT; } printk("Processor %u found.\n", cpu); if (smp_ops->give_timebase) smp_ops->give_timebase(); /* Wait until cpu puts itself in the online map */ while (!cpu_online(cpu)) cpu_relax(); return 0; }
static void __cpuinit smp_callin (void) { int cpuid, phys_id, itc_master; struct cpuinfo_ia64 *last_cpuinfo, *this_cpuinfo; extern void ia64_init_itm(void); extern volatile int time_keeper_id; #ifdef CONFIG_PERFMON extern void pfm_init_percpu(void); #endif cpuid = smp_processor_id(); phys_id = hard_smp_processor_id(); itc_master = time_keeper_id; if (cpu_online(cpuid)) { printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n", phys_id, cpuid); BUG(); } fix_b0_for_bsp(); /* * numa_node_id() works after this. */ set_numa_node(cpu_to_node_map[cpuid]); set_numa_mem(local_memory_node(cpu_to_node_map[cpuid])); spin_lock(&vector_lock); /* Setup the per cpu irq handling data structures */ __setup_vector_irq(cpuid); notify_cpu_starting(cpuid); set_cpu_online(cpuid, true); per_cpu(cpu_state, cpuid) = CPU_ONLINE; spin_unlock(&vector_lock); smp_setup_percpu_timer(); ia64_mca_cmc_vector_setup(); /* Setup vector on AP */ #ifdef CONFIG_PERFMON pfm_init_percpu(); #endif local_irq_enable(); if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) { /* * Synchronize the ITC with the BP. Need to do this after irqs are * enabled because ia64_sync_itc() calls smp_call_function_single(), which * calls spin_unlock_bh(), which calls spin_unlock_bh(), which calls * local_bh_enable(), which bugs out if irqs are not enabled... */ Dprintk("Going to syncup ITC with ITC Master.\n"); ia64_sync_itc(itc_master); } /* * Get our bogomips. */ ia64_init_itm(); /* * Delay calibration can be skipped if new processor is identical to the * previous processor. */ last_cpuinfo = cpu_data(cpuid - 1); this_cpuinfo = local_cpu_data; if (last_cpuinfo->itc_freq != this_cpuinfo->itc_freq || last_cpuinfo->proc_freq != this_cpuinfo->proc_freq || last_cpuinfo->features != this_cpuinfo->features || last_cpuinfo->revision != this_cpuinfo->revision || last_cpuinfo->family != this_cpuinfo->family || last_cpuinfo->archrev != this_cpuinfo->archrev || last_cpuinfo->model != this_cpuinfo->model) calibrate_delay(); local_cpu_data->loops_per_jiffy = loops_per_jiffy; /* * Allow the master to continue. */ cpu_set(cpuid, cpu_callin_map); Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid); }
/* Interrupts are disabled. */ void xics_migrate_irqs_away(void) { int cpu = smp_processor_id(), hw_cpu = hard_smp_processor_id(); unsigned int irq, virq; struct irq_desc *desc; /* If we used to be the default server, move to the new "boot_cpuid" */ if (hw_cpu == xics_default_server) xics_update_irq_servers(); /* Reject any interrupt that was queued to us... */ icp_ops->set_priority(0); /* Remove ourselves from the global interrupt queue */ xics_set_cpu_giq(xics_default_distrib_server, 0); /* Allow IPIs again... */ icp_ops->set_priority(DEFAULT_PRIORITY); for_each_irq_desc(virq, desc) { struct irq_chip *chip; long server; unsigned long flags; struct ics *ics; /* We can't set affinity on ISA interrupts */ if (virq < NUM_ISA_INTERRUPTS) continue; /* We only need to migrate enabled IRQS */ if (!desc->action) continue; if (desc->irq_data.domain != xics_host) continue; irq = desc->irq_data.hwirq; /* We need to get IPIs still. */ if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) continue; chip = irq_desc_get_chip(desc); if (!chip || !chip->irq_set_affinity) continue; raw_spin_lock_irqsave(&desc->lock, flags); /* Locate interrupt server */ server = -1; ics = irq_get_chip_data(virq); if (ics) server = ics->get_server(ics, irq); if (server < 0) { printk(KERN_ERR "%s: Can't find server for irq %d\n", __func__, irq); goto unlock; } /* We only support delivery to all cpus or to one cpu. * The irq has to be migrated only in the single cpu * case. */ if (server != hw_cpu) goto unlock; /* This is expected during cpu offline. */ if (cpu_online(cpu)) pr_warning("IRQ %u affinity broken off cpu %u\n", virq, cpu); /* Reset affinity to all cpus */ raw_spin_unlock_irqrestore(&desc->lock, flags); irq_set_affinity(virq, cpu_all_mask); continue; unlock: raw_spin_unlock_irqrestore(&desc->lock, flags); } }