int memory_poke_kernel_address(const void *addr, word value)
{
struct page *page;
void *page_addr;
if ((word)addr & (sizeof(word) - 1)) {
ERROR(-ERROR_POINT);
}
if (__module_address((unsigned long)addr) == NULL) {
page = virt_to_page(addr);
} else {
page = vmalloc_to_page(addr);
}
if (!page) {
ERROR(-ERROR_POINT);
}
page_addr = vmap(&page, 1, VM_MAP, PAGE_KERNEL);
if (!page_addr) {
ERROR(-ERROR_MEM);
}
*(word *)(page_addr + ((word)addr & (PAGE_SIZE - 1))) = value;
vunmap(page_addr);
return 0;
}
static void jump_label_del_module(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
struct jump_label_key *key = NULL;
struct jump_label_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
key = (struct jump_label_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
prev = &key->next;
jlm = key->next;
while (jlm && jlm->mod != mod) {
prev = &jlm->next;
jlm = jlm->next;
}
if (jlm) {
*prev = jlm->next;
kfree(jlm);
}
}
}
static int jump_label_add_module(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
struct static_key *key = NULL;
struct static_key_mod *jlm, *jlm2;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
return 0;
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
struct static_key *iterk;
iterk = jump_entry_key(iter);
if (iterk == key)
continue;
key = iterk;
if (within_module(iter->key, mod)) {
static_key_set_entries(key, iter);
continue;
}
jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
if (!static_key_linked(key)) {
jlm2 = kzalloc(sizeof(struct static_key_mod),
GFP_KERNEL);
if (!jlm2) {
kfree(jlm);
return -ENOMEM;
}
preempt_disable();
jlm2->mod = __module_address((unsigned long)key);
preempt_enable();
jlm2->entries = static_key_entries(key);
jlm2->next = NULL;
static_key_set_mod(key, jlm2);
static_key_set_linked(key);
}
jlm->mod = mod;
jlm->entries = iter;
jlm->next = static_key_mod(key);
static_key_set_mod(key, jlm);
static_key_set_linked(key);
/* Only update if we've changed from our initial state */
if (jump_label_type(iter) != jump_label_init_type(iter))
__jump_label_update(key, iter, iter_stop);
}
return 0;
}
static void jump_label_update(struct static_key *key, int enable)
{
struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry = jump_label_get_entries(key);
#ifdef CONFIG_MODULES
struct module *mod = __module_address((unsigned long)key);
__jump_label_mod_update(key, enable);
if (mod)
stop = mod->jump_entries + mod->num_jump_entries;
#endif
if (entry)
__jump_label_update(key, entry, stop, enable);
}
static int report_trace(struct stackframe *frame, void *d)
{
struct module *mod;
unsigned int *depth = d, addr = frame->pc, cookie = NO_COOKIE, cpu = smp_processor_id();
if (*depth) {
mod = __module_address(addr);
if (mod) {
cookie = get_cookie(cpu, per_cpu(backtrace_buffer, cpu), current, NULL, mod, true);
addr = addr - (unsigned long)mod->module_core;
}
marshal_backtrace(addr & ~1, cookie);
(*depth)--;
}
return *depth == 0;
}
static int jump_label_add_module(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
struct static_key *key = NULL;
struct static_key_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
return 0;
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
struct static_key *iterk;
iterk = (struct static_key *)(unsigned long)iter->key;
if (iterk == key)
continue;
key = iterk;
if (__module_address(iter->key) == mod) {
/*
* Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
*/
*((unsigned long *)&key->entries) += (unsigned long)iter;
key->next = NULL;
continue;
}
jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
if (jump_label_type(key) == JUMP_LABEL_ENABLE)
__jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE);
}
return 0;
}
static void jump_label_update(struct static_key *key)
{
struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry = static_key_entries(key);
#ifdef CONFIG_MODULES
struct module *mod;
__jump_label_mod_update(key);
preempt_disable();
mod = __module_address((unsigned long)key);
if (mod)
stop = mod->jump_entries + mod->num_jump_entries;
preempt_enable();
#endif
/* if there are no users, entry can be NULL */
if (entry)
__jump_label_update(key, entry, stop);
}
static int jump_label_add_module(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
struct jump_label_key *key = NULL;
struct jump_label_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
return 0;
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
key = (struct jump_label_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod) {
atomic_set(&key->enabled, 0);
key->entries = iter;
key->next = NULL;
continue;
}
jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
if (jump_label_enabled(key))
__jump_label_update(key, iter, JUMP_LABEL_ENABLE);
}
return 0;
}
void add_cookie(struct pt_regs *regs, int cpu)
{
unsigned long pc;
off_t off;
if (regs == 0)
return;
pc = profile_pc(regs);
if (user_mode(regs)) {
struct mm_struct *mm;
struct vm_area_struct *vma;
struct path *ppath;
mm = current->mm;
for (vma = find_vma(mm, s.pc); vma; vma = vma->vm_next) {
if (s.pc < vma->vm_start || s.pc >= vma->vm_end)
continue;
if (vma->vm_file) {
ppath = &(vma->vm_file->f_path);
off = (vma->vm_pgoff << PAGE_SHIFT) + s.pc - vma->vm_start;
cookie(current->comm, pc, (char *)(ppath->dentry->d_name.name), off, 0);
} else {
/* must be an anonymous map */
cookie(current->comm, pc, "nofile", pc, 0);
}
break;
}
} else {
struct module *mod = __module_address(s.pc);
if (mod) {
s.off = s.pc - (unsigned long)mod->module_core;
cookie(current->comm, pc, mod->name, off, 1);
} else {
s.off = 0;
cookie(current->comm, pc, "vmlinux", off, 0);
}
}
}
static int report_trace(struct stackframe *frame, void *d)
{
unsigned int *depth = d, cookie = NO_COOKIE;
unsigned long addr = frame->pc;
if (*depth) {
#if defined(MODULE)
unsigned int cpu = get_physical_cpu();
struct module *mod = __module_address(addr);
if (mod) {
cookie = get_cookie(cpu, current, mod->name, false);
addr = addr - (unsigned long)mod->module_core;
}
#endif
marshal_backtrace(addr & ~1, cookie);
(*depth)--;
}
return *depth == 0;
}
