[deliverable/linux.git] / kernel / jump_label.c

/*
 * jump label support
 *
 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
 *
 */
#include <linux/jump_label.h>
#include <linux/memory.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>

#ifdef HAVE_JUMP_LABEL

#define JUMP_LABEL_HASH_BITS 6
#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];

/* mutex to protect coming/going of the the jump_label table */
static DEFINE_MUTEX(jump_label_mutex);

struct jump_label_entry {
	struct hlist_node hlist;
	struct jump_entry *table;
	int nr_entries;
	/* hang modules off here */
	struct hlist_head modules;
	unsigned long key;
};

struct jump_label_module_entry {
	struct hlist_node hlist;
	struct jump_entry *table;
	int nr_entries;
	struct module *mod;
};

static int jump_label_cmp(const void *a, const void *b)
{
	const struct jump_entry *jea = a;
	const struct jump_entry *jeb = b;

	if (jea->key < jeb->key)
		return -1;

	if (jea->key > jeb->key)
		return 1;

	return 0;
}

static void
sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
{
	unsigned long size;

	size = (((unsigned long)stop - (unsigned long)start)
					/ sizeof(struct jump_entry));
	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}

static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
{
	struct hlist_head *head;
	struct hlist_node *node;
	struct jump_label_entry *e;
	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);

	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
	hlist_for_each_entry(e, node, head, hlist) {
		if (key == e->key)
			return e;
	}
	return NULL;
}

static struct jump_label_entry *
add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
{
	struct hlist_head *head;
	struct jump_label_entry *e;
	u32 hash;

	e = get_jump_label_entry(key);
	if (e)
		return ERR_PTR(-EEXIST);

	e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
	if (!e)
		return ERR_PTR(-ENOMEM);

	hash = jhash((void *)&key, sizeof(jump_label_t), 0);
	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
	e->key = key;
	e->table = table;
	e->nr_entries = nr_entries;
	INIT_HLIST_HEAD(&(e->modules));
	hlist_add_head(&e->hlist, head);
	return e;
}

static int
build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
{
	struct jump_entry *iter, *iter_begin;
	struct jump_label_entry *entry;
	int count;

	sort_jump_label_entries(start, stop);
	iter = start;
	while (iter < stop) {
		entry = get_jump_label_entry(iter->key);
		if (!entry) {
			iter_begin = iter;
			count = 0;
			while ((iter < stop) &&
				(iter->key == iter_begin->key)) {
				iter++;
				count++;
			}
			entry = add_jump_label_entry(iter_begin->key,
							count, iter_begin);
			if (IS_ERR(entry))
				return PTR_ERR(entry);
		 } else {
			WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
			return -1;
		}
	}
	return 0;
}

/***
 * jump_label_update - update jump label text
 * @key -  key value associated with a a jump label
 * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
 *
 * Will enable/disable the jump for jump label @key, depending on the
 * value of @type.
 *
 */

void jump_label_update(unsigned long key, enum jump_label_type type)
{
	struct jump_entry *iter;
	struct jump_label_entry *entry;
	struct hlist_node *module_node;
	struct jump_label_module_entry *e_module;
	int count;

	mutex_lock(&jump_label_mutex);
	entry = get_jump_label_entry((jump_label_t)key);
	if (entry) {
		count = entry->nr_entries;
		iter = entry->table;
		while (count--) {
			if (kernel_text_address(iter->code))
				arch_jump_label_transform(iter, type);
			iter++;
		}
		/* eanble/disable jump labels in modules */
		hlist_for_each_entry(e_module, module_node, &(entry->modules),
							hlist) {
			count = e_module->nr_entries;
			iter = e_module->table;
			while (count--) {
				if (iter->key &&
						kernel_text_address(iter->code))
					arch_jump_label_transform(iter, type);
				iter++;
			}
		}
	}
	mutex_unlock(&jump_label_mutex);
}

static int addr_conflict(struct jump_entry *entry, void *start, void *end)
{
	if (entry->code <= (unsigned long)end &&
		entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
		return 1;

	return 0;
}

#ifdef CONFIG_MODULES

static int module_conflict(void *start, void *end)
{
	struct hlist_head *head;
	struct hlist_node *node, *node_next, *module_node, *module_node_next;
	struct jump_label_entry *e;
	struct jump_label_module_entry *e_module;
	struct jump_entry *iter;
	int i, count;
	int conflict = 0;

	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
		head = &jump_label_table[i];
		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
			hlist_for_each_entry_safe(e_module, module_node,
							module_node_next,
							&(e->modules), hlist) {
				count = e_module->nr_entries;
				iter = e_module->table;
				while (count--) {
					if (addr_conflict(iter, start, end)) {
						conflict = 1;
						goto out;
					}
					iter++;
				}
			}
		}
	}
out:
	return conflict;
}

#endif

/***
 * jump_label_text_reserved - check if addr range is reserved
 * @start: start text addr
 * @end: end text addr
 *
 * checks if the text addr located between @start and @end
 * overlaps with any of the jump label patch addresses. Code
 * that wants to modify kernel text should first verify that
 * it does not overlap with any of the jump label addresses.
 *
 * returns 1 if there is an overlap, 0 otherwise
 */
int jump_label_text_reserved(void *start, void *end)
{
	struct jump_entry *iter;
	struct jump_entry *iter_start = __start___jump_table;
	struct jump_entry *iter_stop = __start___jump_table;
	int conflict = 0;

	mutex_lock(&jump_label_mutex);
	iter = iter_start;
	while (iter < iter_stop) {
		if (addr_conflict(iter, start, end)) {
			conflict = 1;
			goto out;
		}
		iter++;
	}

	/* now check modules */
#ifdef CONFIG_MODULES
	conflict = module_conflict(start, end);
#endif
out:
	mutex_unlock(&jump_label_mutex);
	return conflict;
}

static __init int init_jump_label(void)
{
	int ret;
	struct jump_entry *iter_start = __start___jump_table;
	struct jump_entry *iter_stop = __stop___jump_table;
	struct jump_entry *iter;

	mutex_lock(&jump_label_mutex);
	ret = build_jump_label_hashtable(__start___jump_table,
					 __stop___jump_table);
	iter = iter_start;
	while (iter < iter_stop) {
		arch_jump_label_text_poke_early(iter->code);
		iter++;
	}
	mutex_unlock(&jump_label_mutex);
	return ret;
}
early_initcall(init_jump_label);

#ifdef CONFIG_MODULES

static struct jump_label_module_entry *
add_jump_label_module_entry(struct jump_label_entry *entry,
			    struct jump_entry *iter_begin,
			    int count, struct module *mod)
{
	struct jump_label_module_entry *e;

	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
	if (!e)
		return ERR_PTR(-ENOMEM);
	e->mod = mod;
	e->nr_entries = count;
	e->table = iter_begin;
	hlist_add_head(&e->hlist, &entry->modules);
	return e;
}

static int add_jump_label_module(struct module *mod)
{
	struct jump_entry *iter, *iter_begin;
	struct jump_label_entry *entry;
	struct jump_label_module_entry *module_entry;
	int count;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
		return 0;

	sort_jump_label_entries(mod->jump_entries,
				mod->jump_entries + mod->num_jump_entries);
	iter = mod->jump_entries;
	while (iter < mod->jump_entries + mod->num_jump_entries) {
		entry = get_jump_label_entry(iter->key);
		iter_begin = iter;
		count = 0;
		while ((iter < mod->jump_entries + mod->num_jump_entries) &&
			(iter->key == iter_begin->key)) {
				iter++;
				count++;
		}
		if (!entry) {
			entry = add_jump_label_entry(iter_begin->key, 0, NULL);
			if (IS_ERR(entry))
				return PTR_ERR(entry);
		}
		module_entry = add_jump_label_module_entry(entry, iter_begin,
							   count, mod);
		if (IS_ERR(module_entry))
			return PTR_ERR(module_entry);
	}
	return 0;
}

static void remove_jump_label_module(struct module *mod)
{
	struct hlist_head *head;
	struct hlist_node *node, *node_next, *module_node, *module_node_next;
	struct jump_label_entry *e;
	struct jump_label_module_entry *e_module;
	int i;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
		return;

	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
		head = &jump_label_table[i];
		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
			hlist_for_each_entry_safe(e_module, module_node,
						  module_node_next,
						  &(e->modules), hlist) {
				if (e_module->mod == mod) {
					hlist_del(&e_module->hlist);
					kfree(e_module);
				}
			}
			if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
				hlist_del(&e->hlist);
				kfree(e);
			}
		}
	}
}

static void remove_jump_label_module_init(struct module *mod)
{
	struct hlist_head *head;
	struct hlist_node *node, *node_next, *module_node, *module_node_next;
	struct jump_label_entry *e;
	struct jump_label_module_entry *e_module;
	struct jump_entry *iter;
	int i, count;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
		return;

	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
		head = &jump_label_table[i];
		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
			hlist_for_each_entry_safe(e_module, module_node,
						  module_node_next,
						  &(e->modules), hlist) {
				if (e_module->mod != mod)
					continue;
				count = e_module->nr_entries;
				iter = e_module->table;
				while (count--) {
					if (within_module_init(iter->code, mod))
						iter->key = 0;
					iter++;
				}
			}
		}
	}
}

static int
jump_label_module_notify(struct notifier_block *self, unsigned long val,
			 void *data)
{
	struct module *mod = data;
	int ret = 0;

	switch (val) {
	case MODULE_STATE_COMING:
		mutex_lock(&jump_label_mutex);
		ret = add_jump_label_module(mod);
		if (ret)
			remove_jump_label_module(mod);
		mutex_unlock(&jump_label_mutex);
		break;
	case MODULE_STATE_GOING:
		mutex_lock(&jump_label_mutex);
		remove_jump_label_module(mod);
		mutex_unlock(&jump_label_mutex);
		break;
	case MODULE_STATE_LIVE:
		mutex_lock(&jump_label_mutex);
		remove_jump_label_module_init(mod);
		mutex_unlock(&jump_label_mutex);
		break;
	}
	return ret;
}

/***
 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
 * @mod: module to patch
 *
 * Allow for run-time selection of the optimal nops. Before the module
 * loads patch these with arch_get_jump_label_nop(), which is specified by
 * the arch specific jump label code.
 */
void jump_label_apply_nops(struct module *mod)
{
	struct jump_entry *iter;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
		return;

	iter = mod->jump_entries;
	while (iter < mod->jump_entries + mod->num_jump_entries) {
		arch_jump_label_text_poke_early(iter->code);
		iter++;
	}
}

struct notifier_block jump_label_module_nb = {
	.notifier_call = jump_label_module_notify,
	.priority = 0,
};

static __init int init_jump_label_module(void)
{
	return register_module_notifier(&jump_label_module_nb);
}
early_initcall(init_jump_label_module);

#endif /* CONFIG_MODULES */

#endif
Commit	Line	Data
bf5438fc JB	1	/*
	2	* jump label support
	3	*
	4	* Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
	5	*
	6	*/
	7	#include <linux/jump_label.h>
	8	#include <linux/memory.h>
	9	#include <linux/uaccess.h>
	10	#include <linux/module.h>
	11	#include <linux/list.h>
	12	#include <linux/jhash.h>
	13	#include <linux/slab.h>
	14	#include <linux/sort.h>
	15	#include <linux/err.h>
	16
	17	#ifdef HAVE_JUMP_LABEL
	18
	19	#define JUMP_LABEL_HASH_BITS 6
	20	#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
	21	static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
	22
	23	/* mutex to protect coming/going of the the jump_label table */
	24	static DEFINE_MUTEX(jump_label_mutex);
	25
	26	struct jump_label_entry {
	27	struct hlist_node hlist;
	28	struct jump_entry *table;
	29	int nr_entries;
	30	/* hang modules off here */
	31	struct hlist_head modules;
	32	unsigned long key;
	33	};
	34
	35	struct jump_label_module_entry {
	36	struct hlist_node hlist;
	37	struct jump_entry *table;
	38	int nr_entries;
	39	struct module *mod;
	40	};
	41
	42	static int jump_label_cmp(const void a, const void b)
	43	{
	44	const struct jump_entry *jea = a;
	45	const struct jump_entry *jeb = b;
	46
	47	if (jea->key < jeb->key)
	48	return -1;
	49
	50	if (jea->key > jeb->key)
	51	return 1;
	52
	53	return 0;
	54	}
	55
	56	static void
	57	sort_jump_label_entries(struct jump_entry start, struct jump_entry stop)
	58	{
	59	unsigned long size;
	60
	61	size = (((unsigned long)stop - (unsigned long)start)
	62	/ sizeof(struct jump_entry));
	63	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
	64	}
65
66	static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
67	{
68	struct hlist_head *head;
69	struct hlist_node *node;
70	struct jump_label_entry *e;
71	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
72
73	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
74	hlist_for_each_entry(e, node, head, hlist) {
75	if (key == e->key)
76	return e;
77	}
78	return NULL;
79	}
80
81	static struct jump_label_entry *
82	add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
83	{
84	struct hlist_head *head;
85	struct jump_label_entry *e;
86	u32 hash;
87
88	e = get_jump_label_entry(key);
89	if (e)
90	return ERR_PTR(-EEXIST);
91
92	e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
93	if (!e)
94	return ERR_PTR(-ENOMEM);
95
96	hash = jhash((void *)&key, sizeof(jump_label_t), 0);
97	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
98	e->key = key;
99	e->table = table;
100	e->nr_entries = nr_entries;
101	INIT_HLIST_HEAD(&(e->modules));
102	hlist_add_head(&e->hlist, head);
103	return e;
104	}
105
106	static int
107	build_jump_label_hashtable(struct jump_entry start, struct jump_entry stop)
108	{
109	struct jump_entry iter, iter_begin;
110	struct jump_label_entry *entry;
111	int count;
112
113	sort_jump_label_entries(start, stop);
114	iter = start;
115	while (iter < stop) {
116	entry = get_jump_label_entry(iter->key);
117	if (!entry) {
118	iter_begin = iter;
119	count = 0;
120	while ((iter < stop) &&
121	(iter->key == iter_begin->key)) {
122	iter++;
123	count++;
124	}
125	entry = add_jump_label_entry(iter_begin->key,
126	count, iter_begin);
127	if (IS_ERR(entry))
128	return PTR_ERR(entry);
129	} else {
130	WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
131	return -1;
132	}
133	}
134	return 0;
135	}
136
137	/***
138	* jump_label_update - update jump label text
139	* @key - key value associated with a a jump label
140	* @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
141	*
142	* Will enable/disable the jump for jump label @key, depending on the
143	* value of @type.
144	*
145	*/
146
147	void jump_label_update(unsigned long key, enum jump_label_type type)
148	{
149	struct jump_entry *iter;
150	struct jump_label_entry *entry;
151	struct hlist_node *module_node;
152	struct jump_label_module_entry *e_module;
153	int count;
154
155	mutex_lock(&jump_label_mutex);
156	entry = get_jump_label_entry((jump_label_t)key);
157	if (entry) {
158	count = entry->nr_entries;
159	iter = entry->table;
160	while (count--) {
161	if (kernel_text_address(iter->code))
162	arch_jump_label_transform(iter, type);
163	iter++;
164	}
165	/* eanble/disable jump labels in modules */
166	hlist_for_each_entry(e_module, module_node, &(entry->modules),
167	hlist) {
168	count = e_module->nr_entries;
169	iter = e_module->table;
170	while (count--) {
b842f8fa JB	171	if (iter->key &&
b842f8fa JB	172	kernel_text_address(iter->code))
bf5438fc JB	173	arch_jump_label_transform(iter, type);
	174	iter++;
	175	}
	176	}
	177	}
	178	mutex_unlock(&jump_label_mutex);
	179	}
	180
4c3ef6d7 JB	181	static int addr_conflict(struct jump_entry entry, void start, void *end)
	182	{
	183	if (entry->code <= (unsigned long)end &&
	184	entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
	185	return 1;
	186
	187	return 0;
	188	}
	189
	190	#ifdef CONFIG_MODULES
	191
	192	static int module_conflict(void start, void end)
	193	{
	194	struct hlist_head *head;
	195	struct hlist_node node, node_next, module_node, module_node_next;
	196	struct jump_label_entry *e;
	197	struct jump_label_module_entry *e_module;
	198	struct jump_entry *iter;
	199	int i, count;
	200	int conflict = 0;
	201
	202	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
	203	head = &jump_label_table[i];
	204	hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
	205	hlist_for_each_entry_safe(e_module, module_node,
	206	module_node_next,
	207	&(e->modules), hlist) {
	208	count = e_module->nr_entries;
	209	iter = e_module->table;
	210	while (count--) {
	211	if (addr_conflict(iter, start, end)) {
	212	conflict = 1;
	213	goto out;
	214	}
	215	iter++;
	216	}
	217	}
	218	}
	219	}
	220	out:
	221	return conflict;
	222	}
	223
	224	#endif
	225
	226	/***
	227	* jump_label_text_reserved - check if addr range is reserved
	228	* @start: start text addr
	229	* @end: end text addr
	230	*
	231	* checks if the text addr located between @start and @end
	232	* overlaps with any of the jump label patch addresses. Code
	233	* that wants to modify kernel text should first verify that
	234	* it does not overlap with any of the jump label addresses.
	235	*
	236	* returns 1 if there is an overlap, 0 otherwise
	237	*/
	238	int jump_label_text_reserved(void start, void end)
	239	{
	240	struct jump_entry *iter;
	241	struct jump_entry *iter_start = __start___jump_table;
	242	struct jump_entry *iter_stop = __start___jump_table;
	243	int conflict = 0;
	244
245	mutex_lock(&jump_label_mutex);
246	iter = iter_start;
247	while (iter < iter_stop) {
248	if (addr_conflict(iter, start, end)) {
249	conflict = 1;
250	goto out;
251	}
252	iter++;
253	}
254
255	/* now check modules */
256	#ifdef CONFIG_MODULES
257	conflict = module_conflict(start, end);
258	#endif
259	out:
260	mutex_unlock(&jump_label_mutex);
261	return conflict;
262	}
263
bf5438fc JB	264	static __init int init_jump_label(void)
	265	{
	266	int ret;
	267	struct jump_entry *iter_start = __start___jump_table;
	268	struct jump_entry *iter_stop = __stop___jump_table;
	269	struct jump_entry *iter;
	270
	271	mutex_lock(&jump_label_mutex);
	272	ret = build_jump_label_hashtable(__start___jump_table,
	273	__stop___jump_table);
	274	iter = iter_start;
	275	while (iter < iter_stop) {
	276	arch_jump_label_text_poke_early(iter->code);
	277	iter++;
	278	}
	279	mutex_unlock(&jump_label_mutex);
	280	return ret;
	281	}
	282	early_initcall(init_jump_label);
	283
	284	#ifdef CONFIG_MODULES
	285
	286	static struct jump_label_module_entry *
	287	add_jump_label_module_entry(struct jump_label_entry *entry,
	288	struct jump_entry *iter_begin,
	289	int count, struct module *mod)
	290	{
	291	struct jump_label_module_entry *e;
	292
	293	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
	294	if (!e)
	295	return ERR_PTR(-ENOMEM);
	296	e->mod = mod;
	297	e->nr_entries = count;
	298	e->table = iter_begin;
	299	hlist_add_head(&e->hlist, &entry->modules);
	300	return e;
	301	}
	302
	303	static int add_jump_label_module(struct module *mod)
	304	{
	305	struct jump_entry iter, iter_begin;
	306	struct jump_label_entry *entry;
	307	struct jump_label_module_entry *module_entry;
	308	int count;
	309
	310	/* if the module doesn't have jump label entries, just return */
	311	if (!mod->num_jump_entries)
	312	return 0;
	313
	314	sort_jump_label_entries(mod->jump_entries,
	315	mod->jump_entries + mod->num_jump_entries);
	316	iter = mod->jump_entries;
	317	while (iter < mod->jump_entries + mod->num_jump_entries) {
	318	entry = get_jump_label_entry(iter->key);
	319	iter_begin = iter;
	320	count = 0;
	321	while ((iter < mod->jump_entries + mod->num_jump_entries) &&
	322	(iter->key == iter_begin->key)) {
	323	iter++;
	324	count++;
	325	}
	326	if (!entry) {
	327	entry = add_jump_label_entry(iter_begin->key, 0, NULL);
328	if (IS_ERR(entry))
329	return PTR_ERR(entry);
330	}
331	module_entry = add_jump_label_module_entry(entry, iter_begin,
332	count, mod);
333	if (IS_ERR(module_entry))
334	return PTR_ERR(module_entry);
335	}
336	return 0;
337	}
338
339	static void remove_jump_label_module(struct module *mod)
340	{
341	struct hlist_head *head;
342	struct hlist_node node, node_next, module_node, module_node_next;
343	struct jump_label_entry *e;
344	struct jump_label_module_entry *e_module;
345	int i;
346
347	/* if the module doesn't have jump label entries, just return */
348	if (!mod->num_jump_entries)
349	return;
350
351	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
352	head = &jump_label_table[i];
353	hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
354	hlist_for_each_entry_safe(e_module, module_node,
355	module_node_next,
356	&(e->modules), hlist) {
357	if (e_module->mod == mod) {
358	hlist_del(&e_module->hlist);
359	kfree(e_module);
360	}
361	}
362	if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
363	hlist_del(&e->hlist);
364	kfree(e);
365	}
366	}
367	}
368	}
369
b842f8fa JB	370	static void remove_jump_label_module_init(struct module *mod)
	371	{
	372	struct hlist_head *head;
	373	struct hlist_node node, node_next, module_node, module_node_next;
	374	struct jump_label_entry *e;
	375	struct jump_label_module_entry *e_module;
	376	struct jump_entry *iter;
	377	int i, count;
	378
	379	/* if the module doesn't have jump label entries, just return */
	380	if (!mod->num_jump_entries)
	381	return;
	382
	383	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
	384	head = &jump_label_table[i];
	385	hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
	386	hlist_for_each_entry_safe(e_module, module_node,
	387	module_node_next,
	388	&(e->modules), hlist) {
	389	if (e_module->mod != mod)
	390	continue;
	391	count = e_module->nr_entries;
	392	iter = e_module->table;
	393	while (count--) {
	394	if (within_module_init(iter->code, mod))
	395	iter->key = 0;
	396	iter++;
	397	}
	398	}
	399	}
	400	}
	401	}
	402
bf5438fc JB	403	static int
	404	jump_label_module_notify(struct notifier_block *self, unsigned long val,
	405	void *data)
	406	{
	407	struct module *mod = data;
	408	int ret = 0;
	409
	410	switch (val) {
	411	case MODULE_STATE_COMING:
	412	mutex_lock(&jump_label_mutex);
	413	ret = add_jump_label_module(mod);
	414	if (ret)
	415	remove_jump_label_module(mod);
	416	mutex_unlock(&jump_label_mutex);
	417	break;
	418	case MODULE_STATE_GOING:
	419	mutex_lock(&jump_label_mutex);
	420	remove_jump_label_module(mod);
	421	mutex_unlock(&jump_label_mutex);
	422	break;
b842f8fa JB	423	case MODULE_STATE_LIVE:
	424	mutex_lock(&jump_label_mutex);
	425	remove_jump_label_module_init(mod);
	426	mutex_unlock(&jump_label_mutex);
	427	break;
bf5438fc JB	428	}
	429	return ret;
	430	}
	431
	432	/***
	433	* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
	434	* @mod: module to patch
	435	*
	436	* Allow for run-time selection of the optimal nops. Before the module
	437	* loads patch these with arch_get_jump_label_nop(), which is specified by
	438	* the arch specific jump label code.
	439	*/
	440	void jump_label_apply_nops(struct module *mod)
	441	{
	442	struct jump_entry *iter;
	443
	444	/* if the module doesn't have jump label entries, just return */
	445	if (!mod->num_jump_entries)
	446	return;
	447
	448	iter = mod->jump_entries;
	449	while (iter < mod->jump_entries + mod->num_jump_entries) {
	450	arch_jump_label_text_poke_early(iter->code);
	451	iter++;
	452	}
	453	}
	454
	455	struct notifier_block jump_label_module_nb = {
	456	.notifier_call = jump_label_module_notify,
	457	.priority = 0,
	458	};
	459
	460	static __init int init_jump_label_module(void)
	461	{
	462	return register_module_notifier(&jump_label_module_nb);
	463	}
	464	early_initcall(init_jump_label_module);
	465
	466	#endif /* CONFIG_MODULES */
	467
	468	#endif