[deliverable/linux.git] / mm / kasan / kasan.c

/*
 * This file contains shadow memory manipulation code.
 *
 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
 *
 * Some code borrowed from https://github.com/xairy/kasan-prototype by
 *        Andrey Konovalov <adech.fo@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define DISABLE_BRANCH_PROFILING

#include <linux/export.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kmemleak.h>
#include <linux/linkage.h>
#include <linux/memblock.h>
#include <linux/memory.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/printk.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/kasan.h>

#include "kasan.h"
#include "../slab.h"

/*
 * Poisons the shadow memory for 'size' bytes starting from 'addr'.
 * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
 */
static void kasan_poison_shadow(const void *address, size_t size, u8 value)
{
	void *shadow_start, *shadow_end;

	shadow_start = kasan_mem_to_shadow(address);
	shadow_end = kasan_mem_to_shadow(address + size);

	memset(shadow_start, value, shadow_end - shadow_start);
}

void kasan_unpoison_shadow(const void *address, size_t size)
{
	kasan_poison_shadow(address, size, 0);

	if (size & KASAN_SHADOW_MASK) {
		u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size);
		*shadow = size & KASAN_SHADOW_MASK;
	}
}

static void __kasan_unpoison_stack(struct task_struct *task, void *sp)
{
	void *base = task_stack_page(task);
	size_t size = sp - base;

	kasan_unpoison_shadow(base, size);
}

/* Unpoison the entire stack for a task. */
void kasan_unpoison_task_stack(struct task_struct *task)
{
	__kasan_unpoison_stack(task, task_stack_page(task) + THREAD_SIZE);
}

/* Unpoison the stack for the current task beyond a watermark sp value. */
asmlinkage void kasan_unpoison_remaining_stack(void *sp)
{
	__kasan_unpoison_stack(current, sp);
}

/*
 * All functions below always inlined so compiler could
 * perform better optimizations in each of __asan_loadX/__assn_storeX
 * depending on memory access size X.
 */

static __always_inline bool memory_is_poisoned_1(unsigned long addr)
{
	s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(shadow_value)) {
		s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
		return unlikely(last_accessible_byte >= shadow_value);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_2(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 1))
			return true;

		/*
		 * If single shadow byte covers 2-byte access, we don't
		 * need to do anything more. Otherwise, test the first
		 * shadow byte.
		 */
		if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_4(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 3))
			return true;

		/*
		 * If single shadow byte covers 4-byte access, we don't
		 * need to do anything more. Otherwise, test the first
		 * shadow byte.
		 */
		if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_8(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 7))
			return true;

		/*
		 * If single shadow byte covers 8-byte access, we don't
		 * need to do anything more. Otherwise, test the first
		 * shadow byte.
		 */
		if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_16(unsigned long addr)
{
	u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		u16 shadow_first_bytes = *(u16 *)shadow_addr;

		if (unlikely(shadow_first_bytes))
			return true;

		/*
		 * If two shadow bytes covers 16-byte access, we don't
		 * need to do anything more. Otherwise, test the last
		 * shadow byte.
		 */
		if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
			return false;

		return memory_is_poisoned_1(addr + 15);
	}

	return false;
}

static __always_inline unsigned long bytes_is_zero(const u8 *start,
					size_t size)
{
	while (size) {
		if (unlikely(*start))
			return (unsigned long)start;
		start++;
		size--;
	}

	return 0;
}

static __always_inline unsigned long memory_is_zero(const void *start,
						const void *end)
{
	unsigned int words;
	unsigned long ret;
	unsigned int prefix = (unsigned long)start % 8;

	if (end - start <= 16)
		return bytes_is_zero(start, end - start);

	if (prefix) {
		prefix = 8 - prefix;
		ret = bytes_is_zero(start, prefix);
		if (unlikely(ret))
			return ret;
		start += prefix;
	}

	words = (end - start) / 8;
	while (words) {
		if (unlikely(*(u64 *)start))
			return bytes_is_zero(start, 8);
		start += 8;
		words--;
	}

	return bytes_is_zero(start, (end - start) % 8);
}

static __always_inline bool memory_is_poisoned_n(unsigned long addr,
						size_t size)
{
	unsigned long ret;

	ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
			kasan_mem_to_shadow((void *)addr + size - 1) + 1);

	if (unlikely(ret)) {
		unsigned long last_byte = addr + size - 1;
		s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte);

		if (unlikely(ret != (unsigned long)last_shadow ||
			((long)(last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
			return true;
	}
	return false;
}

static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
{
	if (__builtin_constant_p(size)) {
		switch (size) {
		case 1:
			return memory_is_poisoned_1(addr);
		case 2:
			return memory_is_poisoned_2(addr);
		case 4:
			return memory_is_poisoned_4(addr);
		case 8:
			return memory_is_poisoned_8(addr);
		case 16:
			return memory_is_poisoned_16(addr);
		default:
			BUILD_BUG();
		}
	}

	return memory_is_poisoned_n(addr, size);
}


static __always_inline void check_memory_region(unsigned long addr,
						size_t size, bool write)
{
	if (unlikely(size == 0))
		return;

	if (unlikely((void *)addr <
		kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
		kasan_report(addr, size, write, _RET_IP_);
		return;
	}

	if (likely(!memory_is_poisoned(addr, size)))
		return;

	kasan_report(addr, size, write, _RET_IP_);
}

void __asan_loadN(unsigned long addr, size_t size);
void __asan_storeN(unsigned long addr, size_t size);

#undef memset
void *memset(void *addr, int c, size_t len)
{
	__asan_storeN((unsigned long)addr, len);

	return __memset(addr, c, len);
}

#undef memmove
void *memmove(void *dest, const void *src, size_t len)
{
	__asan_loadN((unsigned long)src, len);
	__asan_storeN((unsigned long)dest, len);

	return __memmove(dest, src, len);
}

#undef memcpy
void *memcpy(void *dest, const void *src, size_t len)
{
	__asan_loadN((unsigned long)src, len);
	__asan_storeN((unsigned long)dest, len);

	return __memcpy(dest, src, len);
}

void kasan_alloc_pages(struct page *page, unsigned int order)
{
	if (likely(!PageHighMem(page)))
		kasan_unpoison_shadow(page_address(page), PAGE_SIZE << order);
}

void kasan_free_pages(struct page *page, unsigned int order)
{
	if (likely(!PageHighMem(page)))
		kasan_poison_shadow(page_address(page),
				PAGE_SIZE << order,
				KASAN_FREE_PAGE);
}

void kasan_poison_slab(struct page *page)
{
	kasan_poison_shadow(page_address(page),
			PAGE_SIZE << compound_order(page),
			KASAN_KMALLOC_REDZONE);
}

void kasan_unpoison_object_data(struct kmem_cache *cache, void *object)
{
	kasan_unpoison_shadow(object, cache->object_size);
}

void kasan_poison_object_data(struct kmem_cache *cache, void *object)
{
	kasan_poison_shadow(object,
			round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),
			KASAN_KMALLOC_REDZONE);
}

void kasan_slab_alloc(struct kmem_cache *cache, void *object)
{
	kasan_kmalloc(cache, object, cache->object_size);
}

void kasan_slab_free(struct kmem_cache *cache, void *object)
{
	unsigned long size = cache->object_size;
	unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE);

	/* RCU slabs could be legally used after free within the RCU period */
	if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
		return;

	kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
}

void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size)
{
	unsigned long redzone_start;
	unsigned long redzone_end;

	if (unlikely(object == NULL))
		return;

	redzone_start = round_up((unsigned long)(object + size),
				KASAN_SHADOW_SCALE_SIZE);
	redzone_end = round_up((unsigned long)object + cache->object_size,
				KASAN_SHADOW_SCALE_SIZE);

	kasan_unpoison_shadow(object, size);
	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
		KASAN_KMALLOC_REDZONE);
}
EXPORT_SYMBOL(kasan_kmalloc);

void kasan_kmalloc_large(const void *ptr, size_t size)
{
	struct page *page;
	unsigned long redzone_start;
	unsigned long redzone_end;

	if (unlikely(ptr == NULL))
		return;

	page = virt_to_page(ptr);
	redzone_start = round_up((unsigned long)(ptr + size),
				KASAN_SHADOW_SCALE_SIZE);
	redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));

	kasan_unpoison_shadow(ptr, size);
	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
		KASAN_PAGE_REDZONE);
}

void kasan_krealloc(const void *object, size_t size)
{
	struct page *page;

	if (unlikely(object == ZERO_SIZE_PTR))
		return;

	page = virt_to_head_page(object);

	if (unlikely(!PageSlab(page)))
		kasan_kmalloc_large(object, size);
	else
		kasan_kmalloc(page->slab_cache, object, size);
}

void kasan_kfree(void *ptr)
{
	struct page *page;

	page = virt_to_head_page(ptr);

	if (unlikely(!PageSlab(page)))
		kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
				KASAN_FREE_PAGE);
	else
		kasan_slab_free(page->slab_cache, ptr);
}

void kasan_kfree_large(const void *ptr)
{
	struct page *page = virt_to_page(ptr);

	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
			KASAN_FREE_PAGE);
}

int kasan_module_alloc(void *addr, size_t size)
{
	void *ret;
	size_t shadow_size;
	unsigned long shadow_start;

	shadow_start = (unsigned long)kasan_mem_to_shadow(addr);
	shadow_size = round_up(size >> KASAN_SHADOW_SCALE_SHIFT,
			PAGE_SIZE);

	if (WARN_ON(!PAGE_ALIGNED(shadow_start)))
		return -EINVAL;

	ret = __vmalloc_node_range(shadow_size, 1, shadow_start,
			shadow_start + shadow_size,
			GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
			PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE,
			__builtin_return_address(0));

	if (ret) {
		find_vm_area(addr)->flags |= VM_KASAN;
		kmemleak_ignore(ret);
		return 0;
	}

	return -ENOMEM;
}

void kasan_free_shadow(const struct vm_struct *vm)
{
	if (vm->flags & VM_KASAN)
		vfree(kasan_mem_to_shadow(vm->addr));
}

static void register_global(struct kasan_global *global)
{
	size_t aligned_size = round_up(global->size, KASAN_SHADOW_SCALE_SIZE);

	kasan_unpoison_shadow(global->beg, global->size);

	kasan_poison_shadow(global->beg + aligned_size,
		global->size_with_redzone - aligned_size,
		KASAN_GLOBAL_REDZONE);
}

void __asan_register_globals(struct kasan_global *globals, size_t size)
{
	int i;

	for (i = 0; i < size; i++)
		register_global(&globals[i]);
}
EXPORT_SYMBOL(__asan_register_globals);

void __asan_unregister_globals(struct kasan_global *globals, size_t size)
{
}
EXPORT_SYMBOL(__asan_unregister_globals);

#define DEFINE_ASAN_LOAD_STORE(size)				\
	void __asan_load##size(unsigned long addr)		\
	{							\
		check_memory_region(addr, size, false);		\
	}							\
	EXPORT_SYMBOL(__asan_load##size);			\
	__alias(__asan_load##size)				\
	void __asan_load##size##_noabort(unsigned long);	\
	EXPORT_SYMBOL(__asan_load##size##_noabort);		\
	void __asan_store##size(unsigned long addr)		\
	{							\
		check_memory_region(addr, size, true);		\
	}							\
	EXPORT_SYMBOL(__asan_store##size);			\
	__alias(__asan_store##size)				\
	void __asan_store##size##_noabort(unsigned long);	\
	EXPORT_SYMBOL(__asan_store##size##_noabort)

DEFINE_ASAN_LOAD_STORE(1);
DEFINE_ASAN_LOAD_STORE(2);
DEFINE_ASAN_LOAD_STORE(4);
DEFINE_ASAN_LOAD_STORE(8);
DEFINE_ASAN_LOAD_STORE(16);

void __asan_loadN(unsigned long addr, size_t size)
{
	check_memory_region(addr, size, false);
}
EXPORT_SYMBOL(__asan_loadN);

__alias(__asan_loadN)
void __asan_loadN_noabort(unsigned long, size_t);
EXPORT_SYMBOL(__asan_loadN_noabort);

void __asan_storeN(unsigned long addr, size_t size)
{
	check_memory_region(addr, size, true);
}
EXPORT_SYMBOL(__asan_storeN);

__alias(__asan_storeN)
void __asan_storeN_noabort(unsigned long, size_t);
EXPORT_SYMBOL(__asan_storeN_noabort);

/* to shut up compiler complaints */
void __asan_handle_no_return(void) {}
EXPORT_SYMBOL(__asan_handle_no_return);

#ifdef CONFIG_MEMORY_HOTPLUG
static int kasan_mem_notifier(struct notifier_block *nb,
			unsigned long action, void *data)
{
	return (action == MEM_GOING_ONLINE) ? NOTIFY_BAD : NOTIFY_OK;
}

static int __init kasan_memhotplug_init(void)
{
	pr_err("WARNING: KASAN doesn't support memory hot-add\n");
	pr_err("Memory hot-add will be disabled\n");

	hotplug_memory_notifier(kasan_mem_notifier, 0);

	return 0;
}

module_init(kasan_memhotplug_init);
#endif
Commit	Line	Data
0b24becc AR	1	/*
	2	* This file contains shadow memory manipulation code.
	3	*
	4	* Copyright (c) 2014 Samsung Electronics Co., Ltd.
2baf9e89	5	* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
0b24becc	6	*
5d0926ef	7	* Some code borrowed from https://github.com/xairy/kasan-prototype by
0b24becc AR	8	* Andrey Konovalov <adech.fo@gmail.com>
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*
	14	*/
	15
	16	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	17	#define DISABLE_BRANCH_PROFILING
	18
	19	#include <linux/export.h>
	20	#include <linux/init.h>
	21	#include <linux/kernel.h>
45937254	22	#include <linux/kmemleak.h>
e3ae1163	23	#include <linux/linkage.h>
0b24becc	24	#include <linux/memblock.h>
786a8959	25	#include <linux/memory.h>
0b24becc	26	#include <linux/mm.h>
bebf56a1	27	#include <linux/module.h>
0b24becc AR	28	#include <linux/printk.h>
	29	#include <linux/sched.h>
	30	#include <linux/slab.h>
	31	#include <linux/stacktrace.h>
	32	#include <linux/string.h>
	33	#include <linux/types.h>
a5af5aa8	34	#include <linux/vmalloc.h>
0b24becc AR	35	#include <linux/kasan.h>
	36
	37	#include "kasan.h"
0316bec2	38	#include "../slab.h"
0b24becc AR	39
	40	/*
	41	* Poisons the shadow memory for 'size' bytes starting from 'addr'.
	42	* Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
	43	*/
	44	static void kasan_poison_shadow(const void *address, size_t size, u8 value)
	45	{
	46	void shadow_start, shadow_end;
	47
	48	shadow_start = kasan_mem_to_shadow(address);
	49	shadow_end = kasan_mem_to_shadow(address + size);
	50
	51	memset(shadow_start, value, shadow_end - shadow_start);
	52	}
	53
	54	void kasan_unpoison_shadow(const void *address, size_t size)
	55	{
	56	kasan_poison_shadow(address, size, 0);
	57
	58	if (size & KASAN_SHADOW_MASK) {
	59	u8 shadow = (u8 )kasan_mem_to_shadow(address + size);
	60	*shadow = size & KASAN_SHADOW_MASK;
	61	}
	62	}
	63
e3ae1163 MR	64	static void __kasan_unpoison_stack(struct task_struct task, void sp)
	65	{
	66	void *base = task_stack_page(task);
	67	size_t size = sp - base;
	68
	69	kasan_unpoison_shadow(base, size);
	70	}
	71
	72	/* Unpoison the entire stack for a task. */
	73	void kasan_unpoison_task_stack(struct task_struct *task)
	74	{
	75	__kasan_unpoison_stack(task, task_stack_page(task) + THREAD_SIZE);
	76	}
	77
	78	/* Unpoison the stack for the current task beyond a watermark sp value. */
	79	asmlinkage void kasan_unpoison_remaining_stack(void *sp)
	80	{
	81	__kasan_unpoison_stack(current, sp);
	82	}
0b24becc AR	83
	84	/*
	85	* All functions below always inlined so compiler could
	86	* perform better optimizations in each of __asan_loadX/__assn_storeX
	87	* depending on memory access size X.
	88	*/
	89
	90	static __always_inline bool memory_is_poisoned_1(unsigned long addr)
	91	{
	92	s8 shadow_value = (s8 )kasan_mem_to_shadow((void *)addr);
	93
	94	if (unlikely(shadow_value)) {
	95	s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
	96	return unlikely(last_accessible_byte >= shadow_value);
	97	}
	98
	99	return false;
	100	}
	101
	102	static __always_inline bool memory_is_poisoned_2(unsigned long addr)
	103	{
	104	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
	105
	106	if (unlikely(*shadow_addr)) {
	107	if (memory_is_poisoned_1(addr + 1))
	108	return true;
	109
10f70262 XQ	110	/*
	111	* If single shadow byte covers 2-byte access, we don't
	112	* need to do anything more. Otherwise, test the first
	113	* shadow byte.
	114	*/
0b24becc AR	115	if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
	116	return false;
	117
	118	return unlikely((u8 )shadow_addr);
	119	}
	120
	121	return false;
	122	}
	123
	124	static __always_inline bool memory_is_poisoned_4(unsigned long addr)
	125	{
	126	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
	127
	128	if (unlikely(*shadow_addr)) {
	129	if (memory_is_poisoned_1(addr + 3))
	130	return true;
	131
10f70262 XQ	132	/*
	133	* If single shadow byte covers 4-byte access, we don't
	134	* need to do anything more. Otherwise, test the first
	135	* shadow byte.
	136	*/
0b24becc AR	137	if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
	138	return false;
	139
	140	return unlikely((u8 )shadow_addr);
	141	}
	142
	143	return false;
	144	}
	145
	146	static __always_inline bool memory_is_poisoned_8(unsigned long addr)
	147	{
	148	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
	149
	150	if (unlikely(*shadow_addr)) {
	151	if (memory_is_poisoned_1(addr + 7))
	152	return true;
	153
10f70262 XQ	154	/*
	155	* If single shadow byte covers 8-byte access, we don't
	156	* need to do anything more. Otherwise, test the first
	157	* shadow byte.
	158	*/
	159	if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
0b24becc AR	160	return false;
	161
	162	return unlikely((u8 )shadow_addr);
	163	}
	164
	165	return false;
	166	}
	167
	168	static __always_inline bool memory_is_poisoned_16(unsigned long addr)
	169	{
	170	u32 shadow_addr = (u32 )kasan_mem_to_shadow((void *)addr);
	171
	172	if (unlikely(*shadow_addr)) {
	173	u16 shadow_first_bytes = (u16 )shadow_addr;
0b24becc AR	174
	175	if (unlikely(shadow_first_bytes))
	176	return true;
	177
10f70262 XQ	178	/*
	179	* If two shadow bytes covers 16-byte access, we don't
	180	* need to do anything more. Otherwise, test the last
	181	* shadow byte.
	182	*/
	183	if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
0b24becc AR	184	return false;
	185
	186	return memory_is_poisoned_1(addr + 15);
	187	}
	188
	189	return false;
	190	}
	191
	192	static __always_inline unsigned long bytes_is_zero(const u8 *start,
	193	size_t size)
	194	{
	195	while (size) {
	196	if (unlikely(*start))
	197	return (unsigned long)start;
	198	start++;
	199	size--;
	200	}
	201
	202	return 0;
	203	}
	204
	205	static __always_inline unsigned long memory_is_zero(const void *start,
	206	const void *end)
	207	{
	208	unsigned int words;
	209	unsigned long ret;
	210	unsigned int prefix = (unsigned long)start % 8;
	211
	212	if (end - start <= 16)
	213	return bytes_is_zero(start, end - start);
	214
	215	if (prefix) {
	216	prefix = 8 - prefix;
	217	ret = bytes_is_zero(start, prefix);
	218	if (unlikely(ret))
	219	return ret;
	220	start += prefix;
	221	}
	222
	223	words = (end - start) / 8;
	224	while (words) {
	225	if (unlikely((u64 )start))
	226	return bytes_is_zero(start, 8);
	227	start += 8;
	228	words--;
	229	}
	230
	231	return bytes_is_zero(start, (end - start) % 8);
	232	}
	233
	234	static __always_inline bool memory_is_poisoned_n(unsigned long addr,
	235	size_t size)
	236	{
	237	unsigned long ret;
	238
	239	ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
	240	kasan_mem_to_shadow((void *)addr + size - 1) + 1);
	241
	242	if (unlikely(ret)) {
	243	unsigned long last_byte = addr + size - 1;
	244	s8 last_shadow = (s8 )kasan_mem_to_shadow((void *)last_byte);
	245
	246	if (unlikely(ret != (unsigned long)last_shadow \|\|
e0d57714	247	((long)(last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
0b24becc AR	248	return true;
	249	}
	250	return false;
	251	}
	252
	253	static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
	254	{
	255	if (__builtin_constant_p(size)) {
	256	switch (size) {
	257	case 1:
	258	return memory_is_poisoned_1(addr);
	259	case 2:
	260	return memory_is_poisoned_2(addr);
	261	case 4:
	262	return memory_is_poisoned_4(addr);
	263	case 8:
	264	return memory_is_poisoned_8(addr);
	265	case 16:
	266	return memory_is_poisoned_16(addr);
	267	default:
	268	BUILD_BUG();
	269	}
	270	}
	271
	272	return memory_is_poisoned_n(addr, size);
	273	}
	274
	275
	276	static __always_inline void check_memory_region(unsigned long addr,
	277	size_t size, bool write)
	278	{
0b24becc AR	279	if (unlikely(size == 0))
	280	return;
	281
	282	if (unlikely((void *)addr <
	283	kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
e9121076	284	kasan_report(addr, size, write, _RET_IP_);
0b24becc AR	285	return;
	286	}
	287
	288	if (likely(!memory_is_poisoned(addr, size)))
	289	return;
	290
	291	kasan_report(addr, size, write, _RET_IP_);
	292	}
	293
393f203f AR	294	void __asan_loadN(unsigned long addr, size_t size);
	295	void __asan_storeN(unsigned long addr, size_t size);
	296
	297	#undef memset
	298	void memset(void addr, int c, size_t len)
	299	{
	300	__asan_storeN((unsigned long)addr, len);
	301
	302	return __memset(addr, c, len);
	303	}
	304
	305	#undef memmove
	306	void memmove(void dest, const void *src, size_t len)
	307	{
	308	__asan_loadN((unsigned long)src, len);
	309	__asan_storeN((unsigned long)dest, len);
	310
	311	return __memmove(dest, src, len);
	312	}
	313
	314	#undef memcpy
	315	void memcpy(void dest, const void *src, size_t len)
	316	{
	317	__asan_loadN((unsigned long)src, len);
	318	__asan_storeN((unsigned long)dest, len);
	319
	320	return __memcpy(dest, src, len);
	321	}
	322
b8c73fc2 AR	323	void kasan_alloc_pages(struct page *page, unsigned int order)
	324	{
	325	if (likely(!PageHighMem(page)))
	326	kasan_unpoison_shadow(page_address(page), PAGE_SIZE << order);
	327	}
	328
	329	void kasan_free_pages(struct page *page, unsigned int order)
	330	{
	331	if (likely(!PageHighMem(page)))
	332	kasan_poison_shadow(page_address(page),
	333	PAGE_SIZE << order,
	334	KASAN_FREE_PAGE);
	335	}
	336
0316bec2 AR	337	void kasan_poison_slab(struct page *page)
	338	{
	339	kasan_poison_shadow(page_address(page),
	340	PAGE_SIZE << compound_order(page),
	341	KASAN_KMALLOC_REDZONE);
	342	}
	343
	344	void kasan_unpoison_object_data(struct kmem_cache cache, void object)
	345	{
	346	kasan_unpoison_shadow(object, cache->object_size);
	347	}
	348
	349	void kasan_poison_object_data(struct kmem_cache cache, void object)
	350	{
	351	kasan_poison_shadow(object,
	352	round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),
	353	KASAN_KMALLOC_REDZONE);
	354	}
	355
	356	void kasan_slab_alloc(struct kmem_cache cache, void object)
	357	{
	358	kasan_kmalloc(cache, object, cache->object_size);
	359	}
	360
	361	void kasan_slab_free(struct kmem_cache cache, void object)
	362	{
	363	unsigned long size = cache->object_size;
	364	unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE);
	365
	366	/* RCU slabs could be legally used after free within the RCU period */
	367	if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
	368	return;
	369
	370	kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
	371	}
	372
	373	void kasan_kmalloc(struct kmem_cache cache, const void object, size_t size)
	374	{
	375	unsigned long redzone_start;
	376	unsigned long redzone_end;
	377
	378	if (unlikely(object == NULL))
	379	return;
	380
	381	redzone_start = round_up((unsigned long)(object + size),
	382	KASAN_SHADOW_SCALE_SIZE);
	383	redzone_end = round_up((unsigned long)object + cache->object_size,
	384	KASAN_SHADOW_SCALE_SIZE);
	385
	386	kasan_unpoison_shadow(object, size);
	387	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
	388	KASAN_KMALLOC_REDZONE);
	389	}
	390	EXPORT_SYMBOL(kasan_kmalloc);
	391
	392	void kasan_kmalloc_large(const void *ptr, size_t size)
	393	{
	394	struct page *page;
	395	unsigned long redzone_start;
	396	unsigned long redzone_end;
	397
	398	if (unlikely(ptr == NULL))
	399	return;
	400
401	page = virt_to_page(ptr);
402	redzone_start = round_up((unsigned long)(ptr + size),
403	KASAN_SHADOW_SCALE_SIZE);
404	redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));
405
406	kasan_unpoison_shadow(ptr, size);
407	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
408	KASAN_PAGE_REDZONE);
409	}
410
411	void kasan_krealloc(const void *object, size_t size)
412	{
413	struct page *page;
414
415	if (unlikely(object == ZERO_SIZE_PTR))
416	return;
417
418	page = virt_to_head_page(object);
419
420	if (unlikely(!PageSlab(page)))
421	kasan_kmalloc_large(object, size);
422	else
423	kasan_kmalloc(page->slab_cache, object, size);
424	}
425
92393615 AR	426	void kasan_kfree(void *ptr)
	427	{
	428	struct page *page;
	429
	430	page = virt_to_head_page(ptr);
	431
	432	if (unlikely(!PageSlab(page)))
	433	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
	434	KASAN_FREE_PAGE);
	435	else
	436	kasan_slab_free(page->slab_cache, ptr);
	437	}
	438
0316bec2 AR	439	void kasan_kfree_large(const void *ptr)
	440	{
	441	struct page *page = virt_to_page(ptr);
	442
	443	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
	444	KASAN_FREE_PAGE);
	445	}
	446
bebf56a1 AR	447	int kasan_module_alloc(void *addr, size_t size)
	448	{
	449	void *ret;
	450	size_t shadow_size;
	451	unsigned long shadow_start;
	452
	453	shadow_start = (unsigned long)kasan_mem_to_shadow(addr);
	454	shadow_size = round_up(size >> KASAN_SHADOW_SCALE_SHIFT,
	455	PAGE_SIZE);
	456
	457	if (WARN_ON(!PAGE_ALIGNED(shadow_start)))
	458	return -EINVAL;
	459
	460	ret = __vmalloc_node_range(shadow_size, 1, shadow_start,
	461	shadow_start + shadow_size,
	462	GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO,
	463	PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE,
	464	__builtin_return_address(0));
a5af5aa8 AR	465
	466	if (ret) {
	467	find_vm_area(addr)->flags \|= VM_KASAN;
45937254	468	kmemleak_ignore(ret);
a5af5aa8 AR	469	return 0;
	470	}
	471
	472	return -ENOMEM;
bebf56a1 AR	473	}
bebf56a1 AR	474
a5af5aa8	475	void kasan_free_shadow(const struct vm_struct *vm)
bebf56a1	476	{
a5af5aa8 AR	477	if (vm->flags & VM_KASAN)
a5af5aa8 AR	478	vfree(kasan_mem_to_shadow(vm->addr));
bebf56a1 AR	479	}
	480
	481	static void register_global(struct kasan_global *global)
	482	{
	483	size_t aligned_size = round_up(global->size, KASAN_SHADOW_SCALE_SIZE);
	484
	485	kasan_unpoison_shadow(global->beg, global->size);
	486
	487	kasan_poison_shadow(global->beg + aligned_size,
	488	global->size_with_redzone - aligned_size,
	489	KASAN_GLOBAL_REDZONE);
	490	}
	491
	492	void __asan_register_globals(struct kasan_global *globals, size_t size)
	493	{
	494	int i;
	495
	496	for (i = 0; i < size; i++)
	497	register_global(&globals[i]);
	498	}
	499	EXPORT_SYMBOL(__asan_register_globals);
	500
	501	void __asan_unregister_globals(struct kasan_global *globals, size_t size)
	502	{
	503	}
	504	EXPORT_SYMBOL(__asan_unregister_globals);
	505
0b24becc AR	506	#define DEFINE_ASAN_LOAD_STORE(size) \
	507	void __asan_load##size(unsigned long addr) \
	508	{ \
	509	check_memory_region(addr, size, false); \
	510	} \
	511	EXPORT_SYMBOL(__asan_load##size); \
	512	__alias(__asan_load##size) \
	513	void __asan_load##size##_noabort(unsigned long); \
	514	EXPORT_SYMBOL(__asan_load##size##_noabort); \
	515	void __asan_store##size(unsigned long addr) \
	516	{ \
	517	check_memory_region(addr, size, true); \
	518	} \
	519	EXPORT_SYMBOL(__asan_store##size); \
	520	__alias(__asan_store##size) \
	521	void __asan_store##size##_noabort(unsigned long); \
	522	EXPORT_SYMBOL(__asan_store##size##_noabort)
	523
	524	DEFINE_ASAN_LOAD_STORE(1);
	525	DEFINE_ASAN_LOAD_STORE(2);
	526	DEFINE_ASAN_LOAD_STORE(4);
	527	DEFINE_ASAN_LOAD_STORE(8);
	528	DEFINE_ASAN_LOAD_STORE(16);
	529
	530	void __asan_loadN(unsigned long addr, size_t size)
	531	{
	532	check_memory_region(addr, size, false);
	533	}
	534	EXPORT_SYMBOL(__asan_loadN);
	535
	536	__alias(__asan_loadN)
	537	void __asan_loadN_noabort(unsigned long, size_t);
	538	EXPORT_SYMBOL(__asan_loadN_noabort);
	539
	540	void __asan_storeN(unsigned long addr, size_t size)
	541	{
	542	check_memory_region(addr, size, true);
	543	}
	544	EXPORT_SYMBOL(__asan_storeN);
	545
	546	__alias(__asan_storeN)
	547	void __asan_storeN_noabort(unsigned long, size_t);
	548	EXPORT_SYMBOL(__asan_storeN_noabort);
	549
	550	/* to shut up compiler complaints */
	551	void __asan_handle_no_return(void) {}
	552	EXPORT_SYMBOL(__asan_handle_no_return);
786a8959 AR	553
	554	#ifdef CONFIG_MEMORY_HOTPLUG
	555	static int kasan_mem_notifier(struct notifier_block *nb,
	556	unsigned long action, void *data)
	557	{
	558	return (action == MEM_GOING_ONLINE) ? NOTIFY_BAD : NOTIFY_OK;
	559	}
	560
	561	static int __init kasan_memhotplug_init(void)
	562	{
25add7ec	563	pr_err("WARNING: KASAN doesn't support memory hot-add\n");
786a8959 AR	564	pr_err("Memory hot-add will be disabled\n");
	565
	566	hotplug_memory_notifier(kasan_mem_notifier, 0);
	567
	568	return 0;
	569	}
	570
	571	module_init(kasan_memhotplug_init);
	572	#endif