/* * Poisons the shadow memory for 'size' bytes starting from 'addr'. * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE. */ static void kasan_poison_shadow(const void *address, size_t size, u8 value) { void *shadow_start, *shadow_end; shadow_start = kasan_mem_to_shadow(address); shadow_end = kasan_mem_to_shadow(address + size); memset(shadow_start, value, shadow_end - shadow_start); }
static void __init map_range(struct range *range) { unsigned long start; unsigned long end; start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); kasan_populate_shadow(start, end, early_pfn_to_nid(range->start)); }
static const void *find_first_bad_addr(const void *addr, size_t size) { u8 shadow_val = *(u8 *)kasan_mem_to_shadow(addr); const void *first_bad_addr = addr; while (!shadow_val && first_bad_addr < addr + size) { first_bad_addr += KASAN_SHADOW_SCALE_SIZE; shadow_val = *(u8 *)kasan_mem_to_shadow(first_bad_addr); } return first_bad_addr; }
static int __init map_range(struct range *range) { unsigned long start; unsigned long end; start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); /* * end + 1 here is intentional. We check several shadow bytes in advance * to slightly speed up fastpath. In some rare cases we could cross * boundary of mapped shadow, so we just map some more here. */ return vmemmap_populate(start, end + 1, NUMA_NO_NODE); }
int kasan_module_alloc(void *addr, size_t size) { void *ret; size_t shadow_size; unsigned long shadow_start; shadow_start = (unsigned long)kasan_mem_to_shadow(addr); shadow_size = round_up(size >> KASAN_SHADOW_SCALE_SHIFT, PAGE_SIZE); if (WARN_ON(!PAGE_ALIGNED(shadow_start))) return -EINVAL; ret = __vmalloc_node_range(shadow_size, 1, shadow_start, shadow_start + shadow_size, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE, __builtin_return_address(0)); if (ret) { find_vm_area(addr)->flags |= VM_KASAN; kmemleak_ignore(ret); return 0; } return -ENOMEM; }
static void print_shadow_for_address(const void *addr) { int i; const void *shadow = kasan_mem_to_shadow(addr); const void *shadow_row; shadow_row = (void *)round_down((unsigned long)shadow, SHADOW_BYTES_PER_ROW) - SHADOW_ROWS_AROUND_ADDR * SHADOW_BYTES_PER_ROW; pr_err("Memory state around the buggy address:\n"); for (i = -SHADOW_ROWS_AROUND_ADDR; i <= SHADOW_ROWS_AROUND_ADDR; i++) { const void *kaddr = kasan_shadow_to_mem(shadow_row); char buffer[4 + (BITS_PER_LONG/8)*2]; snprintf(buffer, sizeof(buffer), (i == 0) ? ">%p: " : " %p: ", kaddr); kasan_disable_current(); print_hex_dump(KERN_ERR, buffer, DUMP_PREFIX_NONE, SHADOW_BYTES_PER_ROW, 1, shadow_row, SHADOW_BYTES_PER_ROW, 0); kasan_enable_current(); if (row_is_guilty(shadow_row, shadow)) pr_err("%*c\n", shadow_pointer_offset(shadow_row, shadow), '^'); shadow_row += SHADOW_BYTES_PER_ROW; } }
static bool __kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip, bool quarantine) { s8 shadow_byte; unsigned long rounded_up_size; if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) != object)) { kasan_report_invalid_free(object, ip); return true; } /* RCU slabs could be legally used after free within the RCU period */ if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU)) return false; shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(object)); if (shadow_byte < 0 || shadow_byte >= KASAN_SHADOW_SCALE_SIZE) { kasan_report_invalid_free(object, ip); return true; } rounded_up_size = round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE); kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE); if (!quarantine || unlikely(!(cache->flags & SLAB_KASAN))) return false; set_track(&get_alloc_info(cache, object)->free_track, GFP_NOWAIT); quarantine_put(get_free_info(cache, object), cache); return true; }
void check_memory_region(unsigned long addr, size_t size, bool write, unsigned long ret_ip) { u8 tag; u8 *shadow_first, *shadow_last, *shadow; void *untagged_addr; if (unlikely(size == 0)) return; tag = get_tag((const void *)addr); /* * Ignore accesses for pointers tagged with 0xff (native kernel * pointer tag) to suppress false positives caused by kmap. * * Some kernel code was written to account for archs that don't keep * high memory mapped all the time, but rather map and unmap particular * pages when needed. Instead of storing a pointer to the kernel memory, * this code saves the address of the page structure and offset within * that page for later use. Those pages are then mapped and unmapped * with kmap/kunmap when necessary and virt_to_page is used to get the * virtual address of the page. For arm64 (that keeps the high memory * mapped all the time), kmap is turned into a page_address call. * The issue is that with use of the page_address + virt_to_page * sequence the top byte value of the original pointer gets lost (gets * set to KASAN_TAG_KERNEL (0xFF)). */ if (tag == KASAN_TAG_KERNEL) return; untagged_addr = reset_tag((const void *)addr); if (unlikely(untagged_addr < kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) { kasan_report(addr, size, write, ret_ip); return; } shadow_first = kasan_mem_to_shadow(untagged_addr); shadow_last = kasan_mem_to_shadow(untagged_addr + size - 1); for (shadow = shadow_first; shadow <= shadow_last; shadow++) { if (*shadow != tag) { kasan_report(addr, size, write, ret_ip); return; } } }
void *find_first_bad_addr(void *addr, size_t size) { void *p = addr; while (p < addr + size && !(*(u8 *)kasan_mem_to_shadow(p))) p += KASAN_SHADOW_SCALE_SIZE; return p; }
void kasan_unpoison_shadow(const void *address, size_t size) { kasan_poison_shadow(address, size, 0); if (size & KASAN_SHADOW_MASK) { u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size); *shadow = size & KASAN_SHADOW_MASK; } }
static __always_inline bool memory_is_poisoned_n(unsigned long addr, size_t size) { unsigned long ret; ret = memory_is_nonzero(kasan_mem_to_shadow((void *)addr), kasan_mem_to_shadow((void *)addr + size - 1) + 1); if (unlikely(ret)) { unsigned long last_byte = addr + size - 1; s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte); if (unlikely(ret != (unsigned long)last_shadow || ((long)(last_byte & KASAN_SHADOW_MASK) >= *last_shadow))) return true; } return false; }
static __always_inline bool memory_is_poisoned_16(unsigned long addr) { u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); /* Unaligned 16-bytes access maps into 3 shadow bytes. */ if (unlikely(!IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) return *shadow_addr || memory_is_poisoned_1(addr + 15); return *shadow_addr; }
void *find_first_bad_addr(void *addr, size_t size) { u8 tag = get_tag(addr); void *p = reset_tag(addr); void *end = p + size; while (p < end && tag == *(u8 *)kasan_mem_to_shadow(p)) p += KASAN_SHADOW_SCALE_SIZE; return p; }
static __always_inline bool memory_is_poisoned_1(unsigned long addr) { s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr); if (unlikely(shadow_value)) { s8 last_accessible_byte = addr & KASAN_SHADOW_MASK; return unlikely(last_accessible_byte >= shadow_value); } return false; }
static void print_error_description(struct kasan_access_info *info) { const char *bug_type = "unknown-crash"; u8 *shadow_addr; info->first_bad_addr = find_first_bad_addr(info->access_addr, info->access_size); shadow_addr = (u8 *)kasan_mem_to_shadow(info->first_bad_addr); /* * If shadow byte value is in [0, KASAN_SHADOW_SCALE_SIZE) we can look * at the next shadow byte to determine the type of the bad access. */ if (*shadow_addr > 0 && *shadow_addr <= KASAN_SHADOW_SCALE_SIZE - 1) shadow_addr++; switch (*shadow_addr) { case 0 ... KASAN_SHADOW_SCALE_SIZE - 1: /* * In theory it's still possible to see these shadow values * due to a data race in the kernel code. */ bug_type = "out-of-bounds"; break; case KASAN_PAGE_REDZONE: case KASAN_KMALLOC_REDZONE: bug_type = "slab-out-of-bounds"; break; case KASAN_GLOBAL_REDZONE: bug_type = "global-out-of-bounds"; break; case KASAN_STACK_LEFT: case KASAN_STACK_MID: case KASAN_STACK_RIGHT: case KASAN_STACK_PARTIAL: bug_type = "stack-out-of-bounds"; break; case KASAN_FREE_PAGE: case KASAN_KMALLOC_FREE: bug_type = "use-after-free"; break; case KASAN_USE_AFTER_SCOPE: bug_type = "use-after-scope"; break; } pr_err("BUG: KASAN: %s in %pS at addr %p\n", bug_type, (void *)info->ip, info->access_addr); pr_err("%s of size %zu by task %s/%d\n", info->is_write ? "Write" : "Read", info->access_size, current->comm, task_pid_nr(current)); }
static const char *get_shadow_bug_type(struct kasan_access_info *info) { const char *bug_type = "unknown-crash"; u8 *shadow_addr; info->first_bad_addr = find_first_bad_addr(info->access_addr, info->access_size); shadow_addr = (u8 *)kasan_mem_to_shadow(info->first_bad_addr); /* * If shadow byte value is in [0, KASAN_SHADOW_SCALE_SIZE) we can look * at the next shadow byte to determine the type of the bad access. */ if (*shadow_addr > 0 && *shadow_addr <= KASAN_SHADOW_SCALE_SIZE - 1) shadow_addr++; switch (*shadow_addr) { case 0 ... KASAN_SHADOW_SCALE_SIZE - 1: /* * In theory it's still possible to see these shadow values * due to a data race in the kernel code. */ bug_type = "out-of-bounds"; break; case KASAN_PAGE_REDZONE: case KASAN_KMALLOC_REDZONE: bug_type = "slab-out-of-bounds"; break; case KASAN_GLOBAL_REDZONE: bug_type = "global-out-of-bounds"; break; case KASAN_STACK_LEFT: case KASAN_STACK_MID: case KASAN_STACK_RIGHT: case KASAN_STACK_PARTIAL: bug_type = "stack-out-of-bounds"; break; case KASAN_FREE_PAGE: case KASAN_KMALLOC_FREE: bug_type = "use-after-free"; break; case KASAN_USE_AFTER_SCOPE: bug_type = "use-after-scope"; break; case KASAN_ALLOCA_LEFT: case KASAN_ALLOCA_RIGHT: bug_type = "alloca-out-of-bounds"; break; } return bug_type; }
static __always_inline bool memory_is_poisoned_2_4_8(unsigned long addr, unsigned long size) { u8 *shadow_addr = (u8 *)kasan_mem_to_shadow((void *)addr); /* * Access crosses 8(shadow size)-byte boundary. Such access maps * into 2 shadow bytes, so we need to check them both. */ if (unlikely(((addr + size - 1) & KASAN_SHADOW_MASK) < size - 1)) return *shadow_addr || memory_is_poisoned_1(addr + size - 1); return memory_is_poisoned_1(addr + size - 1); }
void __init kasan_init(void) { int i; #ifdef CONFIG_KASAN_INLINE register_die_notifier(&kasan_die_notifier); #endif memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt)); load_cr3(early_level4_pgt); __flush_tlb_all(); clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, kasan_mem_to_shadow((void *)PAGE_OFFSET)); for (i = 0; i < E820_X_MAX; i++) { if (pfn_mapped[i].end == 0) break; if (map_range(&pfn_mapped[i])) panic("kasan: unable to allocate shadow!"); } kasan_populate_zero_shadow( kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), kasan_mem_to_shadow((void *)__START_KERNEL_map)); vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext), (unsigned long)kasan_mem_to_shadow(_end), NUMA_NO_NODE); kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), (void *)KASAN_SHADOW_END); load_cr3(init_level4_pgt); __flush_tlb_all(); /* * kasan_zero_page has been used as early shadow memory, thus it may * contain some garbage. Now we can clear and write protect it, since * after the TLB flush no one should write to it. */ memset(kasan_zero_page, 0, PAGE_SIZE); for (i = 0; i < PTRS_PER_PTE; i++) { pte_t pte = __pte(__pa(kasan_zero_page) | __PAGE_KERNEL_RO); set_pte(&kasan_zero_pte[i], pte); } /* Flush TLBs again to be sure that write protection applied. */ __flush_tlb_all(); init_task.kasan_depth = 0; pr_info("KernelAddressSanitizer initialized\n"); }
void __init kasan_init(void) { u64 kimg_shadow_start, kimg_shadow_end; u64 mod_shadow_start, mod_shadow_end; struct memblock_region *reg; int i; kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK; kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end)); mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR); mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END); /* * We are going to perform proper setup of shadow memory. * At first we should unmap early shadow (clear_pgds() call below). * However, instrumented code couldn't execute without shadow memory. * tmp_pg_dir used to keep early shadow mapped until full shadow * setup will be finished. */ memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir)); dsb(ishst); cpu_replace_ttbr1(lm_alias(tmp_pg_dir)); clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); kasan_map_populate(kimg_shadow_start, kimg_shadow_end, early_pfn_to_nid(virt_to_pfn(lm_alias(_text)))); kasan_populate_early_shadow((void *)KASAN_SHADOW_START, (void *)mod_shadow_start); kasan_populate_early_shadow((void *)kimg_shadow_end, kasan_mem_to_shadow((void *)PAGE_OFFSET)); if (kimg_shadow_start > mod_shadow_end) kasan_populate_early_shadow((void *)mod_shadow_end, (void *)kimg_shadow_start); for_each_memblock(memory, reg) { void *start = (void *)__phys_to_virt(reg->base); void *end = (void *)__phys_to_virt(reg->base + reg->size); if (start >= end) break; kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), (unsigned long)kasan_mem_to_shadow(end), early_pfn_to_nid(virt_to_pfn(start))); }
static __always_inline bool memory_is_poisoned_8(unsigned long addr) { u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); if (unlikely(*shadow_addr)) { if (memory_is_poisoned_1(addr + 7)) return true; /* * If single shadow byte covers 8-byte access, we don't * need to do anything more. Otherwise, test the first * shadow byte. */ if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) return false; return unlikely(*(u8 *)shadow_addr); } return false; }
static int __meminit kasan_mem_notifier(struct notifier_block *nb, unsigned long action, void *data) { struct memory_notify *mem_data = data; unsigned long nr_shadow_pages, start_kaddr, shadow_start; unsigned long shadow_end, shadow_size; nr_shadow_pages = mem_data->nr_pages >> KASAN_SHADOW_SCALE_SHIFT; start_kaddr = (unsigned long)pfn_to_kaddr(mem_data->start_pfn); shadow_start = (unsigned long)kasan_mem_to_shadow((void *)start_kaddr); shadow_size = nr_shadow_pages << PAGE_SHIFT; shadow_end = shadow_start + shadow_size; if (WARN_ON(mem_data->nr_pages % KASAN_SHADOW_SCALE_SIZE) || WARN_ON(start_kaddr % (KASAN_SHADOW_SCALE_SIZE << PAGE_SHIFT))) return NOTIFY_BAD; switch (action) { case MEM_GOING_ONLINE: { void *ret; ret = __vmalloc_node_range(shadow_size, PAGE_SIZE, shadow_start, shadow_end, GFP_KERNEL, PAGE_KERNEL, VM_NO_GUARD, pfn_to_nid(mem_data->start_pfn), __builtin_return_address(0)); if (!ret) return NOTIFY_BAD; kmemleak_ignore(ret); return NOTIFY_OK; } case MEM_OFFLINE: vfree((void *)shadow_start); } return NOTIFY_OK; }
bool kasan_slab_free(struct kmem_cache *cache, void *object) { s8 shadow_byte; /* RCU slabs could be legally used after free within the RCU period */ if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU)) return false; shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(object)); if (shadow_byte < 0 || shadow_byte >= KASAN_SHADOW_SCALE_SIZE) { kasan_report_double_free(cache, object, shadow_byte); return true; } kasan_poison_slab_free(cache, object); if (unlikely(!(cache->flags & SLAB_KASAN))) return false; set_track(&get_alloc_info(cache, object)->free_track, GFP_NOWAIT); quarantine_put(get_free_info(cache, object), cache); return true; }
static __always_inline bool memory_is_poisoned_16(unsigned long addr) { u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr); if (unlikely(*shadow_addr)) { u16 shadow_first_bytes = *(u16 *)shadow_addr; if (unlikely(shadow_first_bytes)) return true; /* * If two shadow bytes covers 16-byte access, we don't * need to do anything more. Otherwise, test the last * shadow byte. */ if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) return false; return memory_is_poisoned_1(addr + 15); } return false; }
static void print_shadow_for_address(const void *addr) { int i; const void *shadow = kasan_mem_to_shadow(addr); const void *shadow_row; shadow_row = (void *)round_down((unsigned long)shadow, SHADOW_BYTES_PER_ROW) - SHADOW_ROWS_AROUND_ADDR * SHADOW_BYTES_PER_ROW; pr_err("Memory state around the buggy address:\n"); for (i = -SHADOW_ROWS_AROUND_ADDR; i <= SHADOW_ROWS_AROUND_ADDR; i++) { const void *kaddr = kasan_shadow_to_mem(shadow_row); char buffer[4 + (BITS_PER_LONG/8)*2]; char shadow_buf[SHADOW_BYTES_PER_ROW]; snprintf(buffer, sizeof(buffer), (i == 0) ? ">%p: " : " %p: ", kaddr); /* * We should not pass a shadow pointer to generic * function, because generic functions may try to * access kasan mapping for the passed address. */ memcpy(shadow_buf, shadow_row, SHADOW_BYTES_PER_ROW); print_hex_dump(KERN_ERR, buffer, DUMP_PREFIX_NONE, SHADOW_BYTES_PER_ROW, 1, shadow_buf, SHADOW_BYTES_PER_ROW, 0); if (row_is_guilty(shadow_row, shadow)) pr_err("%*c\n", shadow_pointer_offset(shadow_row, shadow), '^'); shadow_row += SHADOW_BYTES_PER_ROW; } }
static void print_error_description(struct kasan_access_info *info) { const char *bug_type = "unknown crash"; u8 shadow_val; info->first_bad_addr = find_first_bad_addr(info->access_addr, info->access_size); shadow_val = *(u8 *)kasan_mem_to_shadow(info->first_bad_addr); switch (shadow_val) { case KASAN_FREE_PAGE: case KASAN_KMALLOC_FREE: bug_type = "use after free"; break; case KASAN_PAGE_REDZONE: case KASAN_KMALLOC_REDZONE: case KASAN_GLOBAL_REDZONE: case 0 ... KASAN_SHADOW_SCALE_SIZE - 1: bug_type = "out of bounds access"; break; case KASAN_STACK_LEFT: case KASAN_STACK_MID: case KASAN_STACK_RIGHT: case KASAN_STACK_PARTIAL: bug_type = "out of bounds on stack"; break; } pr_err("BUG: KASan: %s in %pS at addr %p\n", bug_type, (void *)info->ip, info->access_addr); pr_err("%s of size %zu by task %s/%d\n", info->is_write ? "Write" : "Read", info->access_size, current->comm, task_pid_nr(current)); }
void print_tags(u8 addr_tag, const void *addr) { u8 *shadow = (u8 *)kasan_mem_to_shadow(addr); pr_err("Pointer tag: [%02x], memory tag: [%02x]\n", addr_tag, *shadow); }
void kasan_free_shadow(const struct vm_struct *vm) { if (vm->flags & VM_KASAN) vfree(kasan_mem_to_shadow(vm->addr)); }
void __init kasan_init(void) { u64 kimg_shadow_start, kimg_shadow_end; u64 mod_shadow_start, mod_shadow_end; struct memblock_region *reg; int i; kimg_shadow_start = (u64)kasan_mem_to_shadow(_text); kimg_shadow_end = (u64)kasan_mem_to_shadow(_end); mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR); mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END); /* * We are going to perform proper setup of shadow memory. * At first we should unmap early shadow (clear_pgds() call bellow). * However, instrumented code couldn't execute without shadow memory. * tmp_pg_dir used to keep early shadow mapped until full shadow * setup will be finished. */ memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir)); dsb(ishst); cpu_replace_ttbr1(lm_alias(tmp_pg_dir)); clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); vmemmap_populate(kimg_shadow_start, kimg_shadow_end, pfn_to_nid(virt_to_pfn(_text))); /* * vmemmap_populate() has populated the shadow region that covers the * kernel image with SWAPPER_BLOCK_SIZE mappings, so we have to round * the start and end addresses to SWAPPER_BLOCK_SIZE as well, to prevent * kasan_populate_zero_shadow() from replacing the page table entries * (PMD or PTE) at the edges of the shadow region for the kernel * image. */ kimg_shadow_start = round_down(kimg_shadow_start, SWAPPER_BLOCK_SIZE); kimg_shadow_end = round_up(kimg_shadow_end, SWAPPER_BLOCK_SIZE); kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, (void *)mod_shadow_start); kasan_populate_zero_shadow((void *)kimg_shadow_end, kasan_mem_to_shadow((void *)PAGE_OFFSET)); if (kimg_shadow_start > mod_shadow_end) kasan_populate_zero_shadow((void *)mod_shadow_end, (void *)kimg_shadow_start); for_each_memblock(memory, reg) { void *start = (void *)__phys_to_virt(reg->base); void *end = (void *)__phys_to_virt(reg->base + reg->size); if (start >= end) break; /* * end + 1 here is intentional. We check several shadow bytes in * advance to slightly speed up fastpath. In some rare cases * we could cross boundary of mapped shadow, so we just map * some more here. */ vmemmap_populate((unsigned long)kasan_mem_to_shadow(start), (unsigned long)kasan_mem_to_shadow(end) + 1, pfn_to_nid(virt_to_pfn(start))); }
void __init kasan_init(void) { int i; void *shadow_cpu_entry_begin, *shadow_cpu_entry_end; #ifdef CONFIG_KASAN_INLINE register_die_notifier(&kasan_die_notifier); #endif memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt)); /* * We use the same shadow offset for 4- and 5-level paging to * facilitate boot-time switching between paging modes. * As result in 5-level paging mode KASAN_SHADOW_START and * KASAN_SHADOW_END are not aligned to PGD boundary. * * KASAN_SHADOW_START doesn't share PGD with anything else. * We claim whole PGD entry to make things easier. * * KASAN_SHADOW_END lands in the last PGD entry and it collides with * bunch of things like kernel code, modules, EFI mapping, etc. * We need to take extra steps to not overwrite them. */ if (pgtable_l5_enabled()) { void *ptr; ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END)); memcpy(tmp_p4d_table, (void *)ptr, sizeof(tmp_p4d_table)); set_pgd(&early_top_pgt[pgd_index(KASAN_SHADOW_END)], __pgd(__pa(tmp_p4d_table) | _KERNPG_TABLE)); } load_cr3(early_top_pgt); __flush_tlb_all(); clear_pgds(KASAN_SHADOW_START & PGDIR_MASK, KASAN_SHADOW_END); kasan_populate_early_shadow((void *)(KASAN_SHADOW_START & PGDIR_MASK), kasan_mem_to_shadow((void *)PAGE_OFFSET)); for (i = 0; i < E820_MAX_ENTRIES; i++) { if (pfn_mapped[i].end == 0) break; map_range(&pfn_mapped[i]); } shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE; shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin); shadow_cpu_entry_begin = (void *)round_down( (unsigned long)shadow_cpu_entry_begin, PAGE_SIZE); shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE); shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end); shadow_cpu_entry_end = (void *)round_up( (unsigned long)shadow_cpu_entry_end, PAGE_SIZE); kasan_populate_early_shadow( kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), shadow_cpu_entry_begin); kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin, (unsigned long)shadow_cpu_entry_end, 0); kasan_populate_early_shadow(shadow_cpu_entry_end, kasan_mem_to_shadow((void *)__START_KERNEL_map)); kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext), (unsigned long)kasan_mem_to_shadow(_end), early_pfn_to_nid(__pa(_stext))); kasan_populate_early_shadow(kasan_mem_to_shadow((void *)MODULES_END), (void *)KASAN_SHADOW_END); load_cr3(init_top_pgt); __flush_tlb_all(); /* * kasan_early_shadow_page has been used as early shadow memory, thus * it may contain some garbage. Now we can clear and write protect it, * since after the TLB flush no one should write to it. */ memset(kasan_early_shadow_page, 0, PAGE_SIZE); for (i = 0; i < PTRS_PER_PTE; i++) { pte_t pte; pgprot_t prot; prot = __pgprot(__PAGE_KERNEL_RO | _PAGE_ENC); pgprot_val(prot) &= __default_kernel_pte_mask; pte = __pte(__pa(kasan_early_shadow_page) | pgprot_val(prot)); set_pte(&kasan_early_shadow_pte[i], pte); } /* Flush TLBs again to be sure that write protection applied. */ __flush_tlb_all(); init_task.kasan_depth = 0; pr_info("KernelAddressSanitizer initialized\n"); }