/* * Check that the idle state is uniform across all CPUs in the CPUidle driver * cpumask */ static bool idle_state_valid(struct device_node *state_node, unsigned int idx, const cpumask_t *cpumask) { int cpu; struct device_node *cpu_node, *curr_state_node; bool valid = true; /* * Compare idle state phandles for index idx on all CPUs in the * CPUidle driver cpumask. Start from next logical cpu following * cpumask_first(cpumask) since that's the CPU state_node was * retrieved from. If a mismatch is found bail out straight * away since we certainly hit a firmware misconfiguration. */ for (cpu = cpumask_next(cpumask_first(cpumask), cpumask); cpu < nr_cpu_ids; cpu = cpumask_next(cpu, cpumask)) { cpu_node = of_cpu_device_node_get(cpu); curr_state_node = of_parse_phandle(cpu_node, "cpu-idle-states", idx); if (state_node != curr_state_node) valid = false; of_node_put(curr_state_node); of_node_put(cpu_node); if (!valid) break; } return valid; }
static void *c_start(struct seq_file *m, loff_t *pos) { *pos = cpumask_next(*pos - 1, cpu_online_mask); if ((*pos) < nr_cpu_ids) return &cpu_data(*pos); return NULL; }
void __init smp_cpus_done(unsigned int max_cpus) { int cpu, next, rc; /* Reset the response to a (now illegal) MSG_START_CPU IPI. */ start_cpu_function_addr = (unsigned long) &panic_start_cpu; cpumask_copy(&init_affinity, cpu_online_mask); /* * Pin ourselves to a single cpu in the initial affinity set * so that kernel mappings for the rootfs are not in the dataplane, * if set, and to avoid unnecessary migrating during bringup. * Use the last cpu just in case the whole chip has been * isolated from the scheduler, to keep init away from likely * more useful user code. This also ensures that work scheduled * via schedule_delayed_work() in the init routines will land * on this cpu. */ for (cpu = cpumask_first(&init_affinity); (next = cpumask_next(cpu, &init_affinity)) < nr_cpu_ids; cpu = next) ; rc = sched_setaffinity(current->pid, cpumask_of(cpu)); if (rc != 0) pr_err("Couldn't set init affinity to cpu %d (%d)\n", cpu, rc); }
static int next_cpu_for_irq(struct irq_data *data) { #ifdef CONFIG_SMP int cpu; int weight = cpumask_weight(data->affinity); if (weight > 1) { cpu = smp_processor_id(); for (;;) { cpu = cpumask_next(cpu, data->affinity); if (cpu >= nr_cpu_ids) { cpu = -1; continue; } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { break; } } } else if (weight == 1) { cpu = cpumask_first(data->affinity); } else { cpu = smp_processor_id(); } return cpu; #else return smp_processor_id(); #endif }
static int show_cpuinfo(struct seq_file *m, void *v) { int n = ptr_to_cpu(v); if (n == 0) { char buf[NR_CPUS*5]; cpulist_scnprintf(buf, sizeof(buf), cpu_online_mask); seq_printf(m, "cpu count\t: %d\n", num_online_cpus()); seq_printf(m, "cpu list\t: %s\n", buf); seq_printf(m, "model name\t: %s\n", chip_model); seq_printf(m, "flags\t\t:\n"); /* nothing for now */ seq_printf(m, "cpu MHz\t\t: %llu.%06llu\n", get_clock_rate() / 1000000, (get_clock_rate() % 1000000)); seq_printf(m, "bogomips\t: %lu.%02lu\n\n", loops_per_jiffy/(500000/HZ), (loops_per_jiffy/(5000/HZ)) % 100); } #ifdef CONFIG_SMP if (!cpu_online(n)) return 0; #endif seq_printf(m, "processor\t: %d\n", n); /* Print only num_online_cpus() blank lines total. */ if (cpumask_next(n, cpu_online_mask) < nr_cpu_ids) seq_printf(m, "\n"); return 0; }
struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s) { struct pcpu_freelist_head *head; struct pcpu_freelist_node *node; unsigned long flags; int orig_cpu, cpu; local_irq_save(flags); orig_cpu = cpu = raw_smp_processor_id(); while (1) { head = per_cpu_ptr(s->freelist, cpu); raw_spin_lock(&head->lock); node = head->first; if (node) { head->first = node->next; raw_spin_unlock_irqrestore(&head->lock, flags); return node; } raw_spin_unlock(&head->lock); cpu = cpumask_next(cpu, cpu_possible_mask); if (cpu >= nr_cpu_ids) cpu = 0; if (cpu == orig_cpu) { local_irq_restore(flags); return NULL; } } }
static int pcrypt_aead_init_tfm(struct crypto_aead *tfm) { int cpu, cpu_index; struct aead_instance *inst = aead_alg_instance(tfm); struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst); struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_aead *cipher; cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) % cpumask_weight(cpu_online_mask); ctx->cb_cpu = cpumask_first(cpu_online_mask); for (cpu = 0; cpu < cpu_index; cpu++) ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask); cipher = crypto_spawn_aead(&ictx->spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); ctx->child = cipher; crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) + sizeof(struct aead_request) + crypto_aead_reqsize(cipher)); return 0; }
static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, struct padata_pcrypt *pcrypt) { unsigned int cpu_index, cpu, i; struct pcrypt_cpumask *cpumask; cpu = *cb_cpu; rcu_read_lock_bh(); cpumask = rcu_dereference(pcrypt->cb_cpumask); if (cpumask_test_cpu(cpu, cpumask->mask)) goto out; if (!cpumask_weight(cpumask->mask)) goto out; cpu_index = cpu % cpumask_weight(cpumask->mask); cpu = cpumask_first(cpumask->mask); for (i = 0; i < cpu_index; i++) cpu = cpumask_next(cpu, cpumask->mask); *cb_cpu = cpu; out: rcu_read_unlock_bh(); return padata_do_parallel(pcrypt->pinst, padata, cpu); }
static void rtas_event_scan(struct work_struct *w) { unsigned int cpu; do_event_scan(); get_online_cpus(); /* raw_ OK because just using CPU as starting point. */ cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask); if (cpu >= nr_cpu_ids) { cpu = cpumask_first(cpu_online_mask); if (first_pass) { first_pass = 0; event_scan_delay = 30*HZ/rtas_event_scan_rate; if (surveillance_timeout != -1) { pr_debug("rtasd: enabling surveillance\n"); enable_surveillance(surveillance_timeout); pr_debug("rtasd: surveillance enabled\n"); } } } schedule_delayed_work_on(cpu, &event_scan_work, __round_jiffies_relative(event_scan_delay, cpu)); put_online_cpus(); }
static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm) { int cpu, cpu_index; struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst); struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct crypto_aead *cipher; ictx->tfm_count++; cpu_index = ictx->tfm_count % cpumask_weight(cpu_online_mask); ctx->cb_cpu = cpumask_first(cpu_online_mask); for (cpu = 0; cpu < cpu_index; cpu++) ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask); cipher = crypto_spawn_aead(crypto_instance_ctx(inst)); if (IS_ERR(cipher)) return PTR_ERR(cipher); ctx->child = cipher; tfm->crt_aead.reqsize = sizeof(struct pcrypt_request) + sizeof(struct aead_givcrypt_request) + crypto_aead_reqsize(cipher); return 0; }
static void *c_start(struct seq_file *m, loff_t *pos) { *pos = cpumask_next(*pos - 1, cpu_online_mask); if ((*pos) < nr_cpu_ids) return (void *)(uintptr_t)(1 + *pos); return NULL; }
/** * cpumask_next_and - get the next cpu in *src1p & *src2p * @n: the cpu prior to the place to search (ie. return will be > @n) * @src1p: the first cpumask pointer * @src2p: the second cpumask pointer * * Returns >= nr_cpu_ids if no further cpus set in both. */ int cpumask_next_and(int n, const struct cpumask *src1p, const struct cpumask *src2p) { while ((n = cpumask_next(n, src1p)) < nr_cpu_ids) if (cpumask_test_cpu(n, src2p)) break; return n; }
static void *c_start(struct seq_file *m, loff_t *pos) { smp_call_function(init_cpu_flags, NULL, 1); *pos = cpumask_next(*pos - 1, cpu_online_mask); if (__cpus_weight(cpu_online_mask, *pos) < num_online_vcpus()) return &cpu_data(*pos); return NULL; }
static void *c_start(struct seq_file *m, loff_t *pos) { if (*pos == 0) /* just in case, cpu 0 is not the first */ *pos = cpumask_first(cpu_online_mask); else *pos = cpumask_next(*pos - 1, cpu_online_mask); if ((*pos) < nr_cpu_ids) return &cpu_data(*pos); return NULL; }
static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) { int cpu, target_cpu; target_cpu = cpumask_first(pd->cpumask.pcpu); for (cpu = 0; cpu < cpu_index; cpu++) target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu); return target_cpu; }
static unsigned int watchdog_next_cpu(unsigned int cpu) { cpumask_t cpus = watchdog_cpus; unsigned int next_cpu; next_cpu = cpumask_next(cpu, &cpus); if (next_cpu >= nr_cpu_ids) next_cpu = cpumask_first(&cpus); if (next_cpu == cpu) return nr_cpu_ids; return next_cpu; }
static inline int find_next_online_cpu(struct ehca_comp_pool *pool) { int cpu; unsigned long flags; WARN_ON_ONCE(!in_interrupt()); if (ehca_debug_level >= 3) ehca_dmp(cpu_online_mask, cpumask_size(), ""); spin_lock_irqsave(&pool->last_cpu_lock, flags); cpu = cpumask_next(pool->last_cpu, cpu_online_mask); if (cpu >= nr_cpu_ids) cpu = cpumask_first(cpu_online_mask); pool->last_cpu = cpu; spin_unlock_irqrestore(&pool->last_cpu_lock, flags); return cpu; }
static void *move_iter(struct timer_list_iter *iter, loff_t offset) { for (; offset; offset--) { iter->cpu = cpumask_next(iter->cpu, cpu_online_mask); if (iter->cpu >= nr_cpu_ids) { #ifdef CONFIG_GENERIC_CLOCKEVENTS if (!iter->second_pass) { iter->cpu = -1; iter->second_pass = true; } else return NULL; #else return NULL; #endif } } return iter; }
static void min_max_constraints_workfunc(struct work_struct *work) { int count = -1; bool up = false; unsigned int cpu; int nr_cpus = num_online_cpus(); int max_cpus = tegra_cpq_max_cpus(); int min_cpus = tegra_cpq_min_cpus(); if (cpq_state == TEGRA_CPQ_DISABLED) return; if (is_lp_cluster()) return; if (nr_cpus < min_cpus) { up = true; count = min_cpus - nr_cpus; } else if (nr_cpus > max_cpus && max_cpus >= min_cpus) { count = nr_cpus - max_cpus; } for (;count > 0; count--) { if (up) { cpu = best_core_to_turn_up(); if (cpu < nr_cpu_ids){ show_status("UP", 0, cpu); cpu_up(cpu); } else break; } else { cpu = cpumask_next(0, cpu_online_mask); if (cpu < nr_cpu_ids){ show_status("DOWN", 0, cpu); cpu_down(cpu); } else break; } } }
static long do_microcode_update(void *_info) { struct microcode_info *info = _info; int error; BUG_ON(info->cpu != smp_processor_id()); error = microcode_update_cpu(info->buffer, info->buffer_size); if ( error ) info->error = error; info->cpu = cpumask_next(info->cpu, &cpu_online_map); if ( info->cpu < nr_cpu_ids ) return continue_hypercall_on_cpu(info->cpu, do_microcode_update, info); error = info->error; xfree(info); return error; }
/* * Try to steal tags from a remote cpu's percpu freelist. * * We first check how many percpu freelists have tags - we don't steal tags * unless enough percpu freelists have tags on them that it's possible more than * half the total tags could be stuck on remote percpu freelists. * * Then we iterate through the cpus until we find some tags - we don't attempt * to find the "best" cpu to steal from, to keep cacheline bouncing to a * minimum. */ static inline void steal_tags(struct percpu_ida *pool, struct percpu_ida_cpu *tags) { unsigned cpus_have_tags, cpu = pool->cpu_last_stolen; struct percpu_ida_cpu *remote; for (cpus_have_tags = cpumask_weight(&pool->cpus_have_tags); cpus_have_tags * IDA_PCPU_SIZE > pool->nr_tags / 2; cpus_have_tags--) { cpu = cpumask_next(cpu, &pool->cpus_have_tags); if (cpu >= nr_cpu_ids) { cpu = cpumask_first(&pool->cpus_have_tags); if (cpu >= nr_cpu_ids) BUG(); } pool->cpu_last_stolen = cpu; remote = per_cpu_ptr(pool->tag_cpu, cpu); cpumask_clear_cpu(cpu, &pool->cpus_have_tags); if (remote == tags) continue; spin_lock(&remote->lock); if (remote->nr_free) { memcpy(tags->freelist, remote->freelist, sizeof(unsigned) * remote->nr_free); tags->nr_free = remote->nr_free; remote->nr_free = 0; } spin_unlock(&remote->lock); if (tags->nr_free) break; } }
/* * This itererator needs some explanation. * It returns 1 for the header position. * This means 2 is cpu 0. * In a hotplugged system some cpus, including cpu 0, may be missing so we have * to use cpumask_* to iterate over the cpus. */ static void *sched_debug_start(struct seq_file *file, loff_t *offset) { unsigned long n = *offset; if (n == 0) return (void *) 1; n--; if (n > 0) n = cpumask_next(n - 1, cpu_online_mask); else n = cpumask_first(cpu_online_mask); *offset = n + 1; if (n < nr_cpu_ids) return (void *)(unsigned long)(n + 2); return NULL; }
static int smp_rescan_cpus_sigp(cpumask_t avail) { int cpu_id, logical_cpu; logical_cpu = cpumask_first(&avail); if (logical_cpu >= nr_cpu_ids) return 0; for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) { if (cpu_known(cpu_id)) continue; __cpu_logical_map[logical_cpu] = cpu_id; smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN; if (!cpu_stopped(logical_cpu)) continue; set_cpu_present(logical_cpu, true); smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED; logical_cpu = cpumask_next(logical_cpu, &avail); if (logical_cpu >= nr_cpu_ids) break; } return 0; }
static void min_max_constraints_workfunc(struct work_struct *work) { int count = -1; bool up = false; unsigned int cpu; int nr_cpus = num_online_cpus(); int max_cpus = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS) ? : 4; int min_cpus = pm_qos_request(PM_QOS_MIN_ONLINE_CPUS); if (cpq_state == TEGRA_CPQ_DISABLED) return; if (is_lp_cluster()) return; if (nr_cpus < min_cpus) { up = true; count = min_cpus - nr_cpus; } else if (nr_cpus > max_cpus && max_cpus >= min_cpus) { count = nr_cpus - max_cpus; } for (;count > 0; count--) { if (up) { cpu = cpumask_next_zero(0, cpu_online_mask); if (cpu < nr_cpu_ids) cpu_up(cpu); else break; } else { cpu = cpumask_next(0, cpu_online_mask); if (cpu < nr_cpu_ids) cpu_down(cpu); else break; } } }
static int smp_rescan_cpus_sclp(cpumask_t avail) { struct sclp_cpu_info *info; int cpu_id, logical_cpu, cpu; int rc; logical_cpu = cpumask_first(&avail); if (logical_cpu >= nr_cpu_ids) return 0; info = kmalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; rc = sclp_get_cpu_info(info); if (rc) goto out; for (cpu = 0; cpu < info->combined; cpu++) { if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type) continue; cpu_id = info->cpu[cpu].address; if (cpu_known(cpu_id)) continue; __cpu_logical_map[logical_cpu] = cpu_id; smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN; set_cpu_present(logical_cpu, true); if (cpu >= info->configured) smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY; else smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED; logical_cpu = cpumask_next(logical_cpu, &avail); if (logical_cpu >= nr_cpu_ids) break; } out: kfree(info); return rc; }
/** * irq_reserve_ipi() - Setup an IPI to destination cpumask * @domain: IPI domain * @dest: cpumask of cpus which can receive the IPI * * Allocate a virq that can be used to send IPI to any CPU in dest mask. * * On success it'll return linux irq number and error code on failure */ int irq_reserve_ipi(struct irq_domain *domain, const struct cpumask *dest) { unsigned int nr_irqs, offset; struct irq_data *data; int virq, i; if (!domain ||!irq_domain_is_ipi(domain)) { pr_warn("Reservation on a non IPI domain\n"); return -EINVAL; } if (!cpumask_subset(dest, cpu_possible_mask)) { pr_warn("Reservation is not in possible_cpu_mask\n"); return -EINVAL; } nr_irqs = cpumask_weight(dest); if (!nr_irqs) { pr_warn("Reservation for empty destination mask\n"); return -EINVAL; } if (irq_domain_is_ipi_single(domain)) { /* * If the underlying implementation uses a single HW irq on * all cpus then we only need a single Linux irq number for * it. We have no restrictions vs. the destination mask. The * underlying implementation can deal with holes nicely. */ nr_irqs = 1; offset = 0; } else { unsigned int next; /* * The IPI requires a seperate HW irq on each CPU. We require * that the destination mask is consecutive. If an * implementation needs to support holes, it can reserve * several IPI ranges. */ offset = cpumask_first(dest); /* * Find a hole and if found look for another set bit after the * hole. For now we don't support this scenario. */ next = cpumask_next_zero(offset, dest); if (next < nr_cpu_ids) next = cpumask_next(next, dest); if (next < nr_cpu_ids) { pr_warn("Destination mask has holes\n"); return -EINVAL; } } virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE); if (virq <= 0) { pr_warn("Can't reserve IPI, failed to alloc descs\n"); return -ENOMEM; } virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, (void *) dest, true); if (virq <= 0) { pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); goto free_descs; } for (i = 0; i < nr_irqs; i++) { data = irq_get_irq_data(virq + i); cpumask_copy(data->common->affinity, dest); data->common->ipi_offset = offset; irq_set_status_flags(virq + i, IRQ_NO_BALANCING); } return virq; free_descs: irq_free_descs(virq, nr_irqs); return -EBUSY; }
/* * This maps the physical memory to kernel virtual address space, a total * of max_low_pfn pages, by creating page tables starting from address * PAGE_OFFSET. * * This routine transitions us from using a set of compiled-in large * pages to using some more precise caching, including removing access * to code pages mapped at PAGE_OFFSET (executed only at MEM_SV_START) * marking read-only data as locally cacheable, striping the remaining * .data and .bss across all the available tiles, and removing access * to pages above the top of RAM (thus ensuring a page fault from a bad * virtual address rather than a hypervisor shoot down for accessing * memory outside the assigned limits). */ static void __init kernel_physical_mapping_init(pgd_t *pgd_base) { unsigned long long irqmask; unsigned long address, pfn; pmd_t *pmd; pte_t *pte; int pte_ofs; const struct cpumask *my_cpu_mask = cpumask_of(smp_processor_id()); struct cpumask kstripe_mask; int rc, i; #if CHIP_HAS_CBOX_HOME_MAP() if (ktext_arg_seen && ktext_hash) { pr_warning("warning: \"ktext\" boot argument ignored" " if \"kcache_hash\" sets up text hash-for-home\n"); ktext_small = 0; } if (kdata_arg_seen && kdata_hash) { pr_warning("warning: \"kdata\" boot argument ignored" " if \"kcache_hash\" sets up data hash-for-home\n"); } if (kdata_huge && !hash_default) { pr_warning("warning: disabling \"kdata=huge\"; requires" " kcache_hash=all or =allbutstack\n"); kdata_huge = 0; } #endif /* * Set up a mask for cpus to use for kernel striping. * This is normally all cpus, but minus dataplane cpus if any. * If the dataplane covers the whole chip, we stripe over * the whole chip too. */ cpumask_copy(&kstripe_mask, cpu_possible_mask); if (!kdata_arg_seen) kdata_mask = kstripe_mask; /* Allocate and fill in L2 page tables */ for (i = 0; i < MAX_NUMNODES; ++i) { #ifdef CONFIG_HIGHMEM unsigned long end_pfn = node_lowmem_end_pfn[i]; #else unsigned long end_pfn = node_end_pfn[i]; #endif unsigned long end_huge_pfn = 0; /* Pre-shatter the last huge page to allow per-cpu pages. */ if (kdata_huge) end_huge_pfn = end_pfn - (HPAGE_SIZE >> PAGE_SHIFT); pfn = node_start_pfn[i]; /* Allocate enough memory to hold L2 page tables for node. */ init_prealloc_ptes(i, end_pfn - pfn); address = (unsigned long) pfn_to_kaddr(pfn); while (pfn < end_pfn) { BUG_ON(address & (HPAGE_SIZE-1)); pmd = get_pmd(pgtables, address); pte = get_prealloc_pte(pfn); if (pfn < end_huge_pfn) { pgprot_t prot = init_pgprot(address); *(pte_t *)pmd = pte_mkhuge(pfn_pte(pfn, prot)); for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE; pfn++, pte_ofs++, address += PAGE_SIZE) pte[pte_ofs] = pfn_pte(pfn, prot); } else { if (kdata_huge) printk(KERN_DEBUG "pre-shattered huge" " page at %#lx\n", address); for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE; pfn++, pte_ofs++, address += PAGE_SIZE) { pgprot_t prot = init_pgprot(address); pte[pte_ofs] = pfn_pte(pfn, prot); } assign_pte(pmd, pte); } } } /* * Set or check ktext_map now that we have cpu_possible_mask * and kstripe_mask to work with. */ if (ktext_all) cpumask_copy(&ktext_mask, cpu_possible_mask); else if (ktext_nondataplane) ktext_mask = kstripe_mask; else if (!cpumask_empty(&ktext_mask)) { /* Sanity-check any mask that was requested */ struct cpumask bad; cpumask_andnot(&bad, &ktext_mask, cpu_possible_mask); cpumask_and(&ktext_mask, &ktext_mask, cpu_possible_mask); if (!cpumask_empty(&bad)) { char buf[NR_CPUS * 5]; cpulist_scnprintf(buf, sizeof(buf), &bad); pr_info("ktext: not using unavailable cpus %s\n", buf); } if (cpumask_empty(&ktext_mask)) { pr_warning("ktext: no valid cpus; caching on %d.\n", smp_processor_id()); cpumask_copy(&ktext_mask, cpumask_of(smp_processor_id())); } } address = MEM_SV_INTRPT; pmd = get_pmd(pgtables, address); pfn = 0; /* code starts at PA 0 */ if (ktext_small) { /* Allocate an L2 PTE for the kernel text */ int cpu = 0; pgprot_t prot = construct_pgprot(PAGE_KERNEL_EXEC, PAGE_HOME_IMMUTABLE); if (ktext_local) { if (ktext_nocache) prot = hv_pte_set_mode(prot, HV_PTE_MODE_UNCACHED); else prot = hv_pte_set_mode(prot, HV_PTE_MODE_CACHE_NO_L3); } else { prot = hv_pte_set_mode(prot, HV_PTE_MODE_CACHE_TILE_L3); cpu = cpumask_first(&ktext_mask); prot = ktext_set_nocache(prot); } BUG_ON(address != (unsigned long)_stext); pte = NULL; for (; address < (unsigned long)_einittext; pfn++, address += PAGE_SIZE) { pte_ofs = pte_index(address); if (pte_ofs == 0) { if (pte) assign_pte(pmd++, pte); pte = alloc_pte(); } if (!ktext_local) { prot = set_remote_cache_cpu(prot, cpu); cpu = cpumask_next(cpu, &ktext_mask); if (cpu == NR_CPUS) cpu = cpumask_first(&ktext_mask); } pte[pte_ofs] = pfn_pte(pfn, prot); } if (pte) assign_pte(pmd, pte); } else { pte_t pteval = pfn_pte(0, PAGE_KERNEL_EXEC); pteval = pte_mkhuge(pteval); #if CHIP_HAS_CBOX_HOME_MAP() if (ktext_hash) { pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_CACHE_HASH_L3); pteval = ktext_set_nocache(pteval); } else #endif /* CHIP_HAS_CBOX_HOME_MAP() */ if (cpumask_weight(&ktext_mask) == 1) { pteval = set_remote_cache_cpu(pteval, cpumask_first(&ktext_mask)); pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_CACHE_TILE_L3); pteval = ktext_set_nocache(pteval); } else if (ktext_nocache) pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_UNCACHED); else pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_CACHE_NO_L3); for (; address < (unsigned long)_einittext; pfn += PFN_DOWN(HPAGE_SIZE), address += HPAGE_SIZE) *(pte_t *)(pmd++) = pfn_pte(pfn, pteval); } /* Set swapper_pgprot here so it is flushed to memory right away. */ swapper_pgprot = init_pgprot((unsigned long)swapper_pg_dir); /* * Since we may be changing the caching of the stack and page * table itself, we invoke an assembly helper to do the * following steps: * * - flush the cache so we start with an empty slate * - install pgtables[] as the real page table * - flush the TLB so the new page table takes effect */ irqmask = interrupt_mask_save_mask(); interrupt_mask_set_mask(-1ULL); rc = flush_and_install_context(__pa(pgtables), init_pgprot((unsigned long)pgtables), __get_cpu_var(current_asid), cpumask_bits(my_cpu_mask)); interrupt_mask_restore_mask(irqmask); BUG_ON(rc != 0); /* Copy the page table back to the normal swapper_pg_dir. */ memcpy(pgd_base, pgtables, sizeof(pgtables)); __install_page_table(pgd_base, __get_cpu_var(current_asid), swapper_pgprot); /* * We just read swapper_pgprot and thus brought it into the cache, * with its new home & caching mode. When we start the other CPUs, * they're going to reference swapper_pgprot via their initial fake * VA-is-PA mappings, which cache everything locally. At that * time, if it's in our cache with a conflicting home, the * simulator's coherence checker will complain. So, flush it out * of our cache; we're not going to ever use it again anyway. */ __insn_finv(&swapper_pgprot); }
/* * For a given kernel data VA, how should it be cached? * We return the complete pgprot_t with caching bits set. */ static pgprot_t __init init_pgprot(ulong address) { int cpu; unsigned long page; enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET }; #if CHIP_HAS_CBOX_HOME_MAP() /* For kdata=huge, everything is just hash-for-home. */ if (kdata_huge) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); #endif /* We map the aliased pages of permanent text inaccessible. */ if (address < (ulong) _sinittext - CODE_DELTA) return PAGE_NONE; /* * We map read-only data non-coherent for performance. We could * use neighborhood caching on TILE64, but it's not clear it's a win. */ if ((address >= (ulong) __start_rodata && address < (ulong) __end_rodata) || address == (ulong) empty_zero_page) { return construct_pgprot(PAGE_KERNEL_RO, PAGE_HOME_IMMUTABLE); } #ifndef __tilegx__ #if !ATOMIC_LOCKS_FOUND_VIA_TABLE() /* Force the atomic_locks[] array page to be hash-for-home. */ if (address == (ulong) atomic_locks) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); #endif #endif /* * Everything else that isn't data or bss is heap, so mark it * with the initial heap home (hash-for-home, or this cpu). This * includes any addresses after the loaded image and any address before * _einitdata, since we already captured the case of text before * _sinittext, and __pa(einittext) is approximately __pa(sinitdata). * * All the LOWMEM pages that we mark this way will get their * struct page homecache properly marked later, in set_page_homes(). * The HIGHMEM pages we leave with a default zero for their * homes, but with a zero free_time we don't have to actually * do a flush action the first time we use them, either. */ if (address >= (ulong) _end || address < (ulong) _einitdata) return construct_pgprot(PAGE_KERNEL, initial_heap_home()); #if CHIP_HAS_CBOX_HOME_MAP() /* Use hash-for-home if requested for data/bss. */ if (kdata_hash) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); #endif /* * Make the w1data homed like heap to start with, to avoid * making it part of the page-striped data area when we're just * going to convert it to read-only soon anyway. */ if (address >= (ulong)__w1data_begin && address < (ulong)__w1data_end) return construct_pgprot(PAGE_KERNEL, initial_heap_home()); /* * Otherwise we just hand out consecutive cpus. To avoid * requiring this function to hold state, we just walk forward from * _sdata by PAGE_SIZE, skipping the readonly and init data, to reach * the requested address, while walking cpu home around kdata_mask. * This is typically no more than a dozen or so iterations. */ page = (((ulong)__w1data_end) + PAGE_SIZE - 1) & PAGE_MASK; BUG_ON(address < page || address >= (ulong)_end); cpu = cpumask_first(&kdata_mask); for (; page < address; page += PAGE_SIZE) { if (page >= (ulong)&init_thread_union && page < (ulong)&init_thread_union + THREAD_SIZE) continue; if (page == (ulong)empty_zero_page) continue; #ifndef __tilegx__ #if !ATOMIC_LOCKS_FOUND_VIA_TABLE() if (page == (ulong)atomic_locks) continue; #endif #endif cpu = cpumask_next(cpu, &kdata_mask); if (cpu == NR_CPUS) cpu = cpumask_first(&kdata_mask); } return construct_pgprot(PAGE_KERNEL, cpu); }
/* * For a given kernel data VA, how should it be cached? * We return the complete pgprot_t with caching bits set. */ static pgprot_t __init init_pgprot(ulong address) { int cpu; unsigned long page; enum { CODE_DELTA = MEM_SV_START - PAGE_OFFSET }; /* For kdata=huge, everything is just hash-for-home. */ if (kdata_huge) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); /* * We map the aliased pages of permanent text so we can * update them if necessary, for ftrace, etc. */ if (address < (ulong) _sinittext - CODE_DELTA) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); /* We map read-only data non-coherent for performance. */ if ((address >= (ulong) __start_rodata && address < (ulong) __end_rodata) || address == (ulong) empty_zero_page) { return construct_pgprot(PAGE_KERNEL_RO, PAGE_HOME_IMMUTABLE); } #ifndef __tilegx__ /* Force the atomic_locks[] array page to be hash-for-home. */ if (address == (ulong) atomic_locks) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); #endif /* * Everything else that isn't data or bss is heap, so mark it * with the initial heap home (hash-for-home, or this cpu). This * includes any addresses after the loaded image and any address before * __init_end, since we already captured the case of text before * _sinittext, and __pa(einittext) is approximately __pa(__init_begin). * * All the LOWMEM pages that we mark this way will get their * struct page homecache properly marked later, in set_page_homes(). * The HIGHMEM pages we leave with a default zero for their * homes, but with a zero free_time we don't have to actually * do a flush action the first time we use them, either. */ if (address >= (ulong) _end || address < (ulong) __init_end) return construct_pgprot(PAGE_KERNEL, initial_heap_home()); /* Use hash-for-home if requested for data/bss. */ if (kdata_hash) return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH); /* * Otherwise we just hand out consecutive cpus. To avoid * requiring this function to hold state, we just walk forward from * __end_rodata by PAGE_SIZE, skipping the readonly and init data, to * reach the requested address, while walking cpu home around * kdata_mask. This is typically no more than a dozen or so iterations. */ page = (((ulong)__end_rodata) + PAGE_SIZE - 1) & PAGE_MASK; BUG_ON(address < page || address >= (ulong)_end); cpu = cpumask_first(&kdata_mask); for (; page < address; page += PAGE_SIZE) { if (page >= (ulong)&init_thread_union && page < (ulong)&init_thread_union + THREAD_SIZE) continue; if (page == (ulong)empty_zero_page) continue; #ifndef __tilegx__ if (page == (ulong)atomic_locks) continue; #endif cpu = cpumask_next(cpu, &kdata_mask); if (cpu == NR_CPUS) cpu = cpumask_first(&kdata_mask); } return construct_pgprot(PAGE_KERNEL, cpu); }
struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, int *cpu, u32 *sh_desc) { size_t size; u32 num_words; dma_addr_t hwdesc; struct caam_drv_ctx *drv_ctx; const cpumask_t *cpus = qman_affine_cpus(); static DEFINE_PER_CPU(int, last_cpu); num_words = desc_len(sh_desc); if (num_words > MAX_SDLEN) { dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); return ERR_PTR(-EINVAL); } drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC); if (!drv_ctx) return ERR_PTR(-ENOMEM); /* * Initialise pre-header - set RSLS and SDLEN - and shared descriptor * and dma-map them. */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, DMA_BIDIRECTIONAL); if (dma_mapping_error(qidev, hwdesc)) { dev_err(qidev, "DMA map error for preheader + shdesc\n"); kfree(drv_ctx); return ERR_PTR(-ENOMEM); } drv_ctx->context_a = hwdesc; /* If given CPU does not own the portal, choose another one that does */ if (!cpumask_test_cpu(*cpu, cpus)) { int *pcpu = &get_cpu_var(last_cpu); *pcpu = cpumask_next(*pcpu, cpus); if (*pcpu >= nr_cpu_ids) *pcpu = cpumask_first(cpus); *cpu = *pcpu; put_cpu_var(last_cpu); } drv_ctx->cpu = *cpu; /* Find response FQ hooked with this CPU */ drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); /* Attach request FQ */ drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, QMAN_INITFQ_FLAG_SCHED); if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) { dev_err(qidev, "create_caam_req_fq failed\n"); dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); kfree(drv_ctx); return ERR_PTR(-ENOMEM); } drv_ctx->qidev = qidev; return drv_ctx; }