2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * struct klp_ops - structure for tracking registered ftrace ops structs
35 * A single ftrace_ops is shared between all enabled replacement functions
36 * (klp_func structs) which have the same old_addr. This allows the switch
37 * between function versions to happen instantaneously by updating the klp_ops
38 * struct's func_stack list. The winner is the klp_func at the top of the
39 * func_stack (front of the list).
41 * @node: node for the global klp_ops list
42 * @func_stack: list head for the stack of klp_func's (active func is on top)
43 * @fops: registered ftrace ops struct
46 struct list_head node
;
47 struct list_head func_stack
;
48 struct ftrace_ops fops
;
52 * The klp_mutex protects the global lists and state transitions of any
53 * structure reachable from them. References to any structure must be obtained
54 * under mutex protection (except in klp_ftrace_handler(), which uses RCU to
55 * ensure it gets consistent data).
57 static DEFINE_MUTEX(klp_mutex
);
59 static LIST_HEAD(klp_patches
);
60 static LIST_HEAD(klp_ops
);
62 static struct kobject
*klp_root_kobj
;
64 static struct klp_ops
*klp_find_ops(unsigned long old_addr
)
67 struct klp_func
*func
;
69 list_for_each_entry(ops
, &klp_ops
, node
) {
70 func
= list_first_entry(&ops
->func_stack
, struct klp_func
,
72 if (func
->old_addr
== old_addr
)
79 static bool klp_is_module(struct klp_object
*obj
)
84 static bool klp_is_object_loaded(struct klp_object
*obj
)
86 return !obj
->name
|| obj
->mod
;
89 /* sets obj->mod if object is not vmlinux and module is found */
90 static void klp_find_object_module(struct klp_object
*obj
)
94 if (!klp_is_module(obj
))
97 mutex_lock(&module_mutex
);
99 * We do not want to block removal of patched modules and therefore
100 * we do not take a reference here. The patches are removed by
101 * a going module handler instead.
103 mod
= find_module(obj
->name
);
105 * Do not mess work of the module coming and going notifiers.
106 * Note that the patch might still be needed before the going handler
107 * is called. Module functions can be called even in the GOING state
108 * until mod->exit() finishes. This is especially important for
109 * patches that modify semantic of the functions.
111 if (mod
&& mod
->klp_alive
)
114 mutex_unlock(&module_mutex
);
117 /* klp_mutex must be held by caller */
118 static bool klp_is_patch_registered(struct klp_patch
*patch
)
120 struct klp_patch
*mypatch
;
122 list_for_each_entry(mypatch
, &klp_patches
, list
)
123 if (mypatch
== patch
)
129 static bool klp_initialized(void)
131 return !!klp_root_kobj
;
134 struct klp_find_arg
{
139 * If count == 0, the symbol was not found. If count == 1, a unique
140 * match was found and addr is set. If count > 1, there is
141 * unresolvable ambiguity among "count" number of symbols with the same
142 * name in the same object.
147 static int klp_find_callback(void *data
, const char *name
,
148 struct module
*mod
, unsigned long addr
)
150 struct klp_find_arg
*args
= data
;
152 if ((mod
&& !args
->objname
) || (!mod
&& args
->objname
))
155 if (strcmp(args
->name
, name
))
158 if (args
->objname
&& strcmp(args
->objname
, mod
->name
))
162 * args->addr might be overwritten if another match is found
163 * but klp_find_object_symbol() handles this and only returns the
164 * addr if count == 1.
172 static int klp_find_object_symbol(const char *objname
, const char *name
,
175 struct klp_find_arg args
= {
182 mutex_lock(&module_mutex
);
183 kallsyms_on_each_symbol(klp_find_callback
, &args
);
184 mutex_unlock(&module_mutex
);
187 pr_err("symbol '%s' not found in symbol table\n", name
);
188 else if (args
.count
> 1)
189 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
190 args
.count
, name
, objname
);
200 struct klp_verify_args
{
202 const unsigned long addr
;
205 static int klp_verify_callback(void *data
, const char *name
,
206 struct module
*mod
, unsigned long addr
)
208 struct klp_verify_args
*args
= data
;
211 !strcmp(args
->name
, name
) &&
218 static int klp_verify_vmlinux_symbol(const char *name
, unsigned long addr
)
220 struct klp_verify_args args
= {
226 mutex_lock(&module_mutex
);
227 ret
= kallsyms_on_each_symbol(klp_verify_callback
, &args
);
228 mutex_unlock(&module_mutex
);
231 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?\n",
239 static int klp_find_verify_func_addr(struct klp_object
*obj
,
240 struct klp_func
*func
)
244 #if defined(CONFIG_RANDOMIZE_BASE)
245 /* If KASLR has been enabled, adjust old_addr accordingly */
246 if (kaslr_enabled() && func
->old_addr
)
247 func
->old_addr
+= kaslr_offset();
250 if (!func
->old_addr
|| klp_is_module(obj
))
251 ret
= klp_find_object_symbol(obj
->name
, func
->old_name
,
254 ret
= klp_verify_vmlinux_symbol(func
->old_name
,
261 * external symbols are located outside the parent object (where the parent
262 * object is either vmlinux or the kmod being patched).
264 static int klp_find_external_symbol(struct module
*pmod
, const char *name
,
267 const struct kernel_symbol
*sym
;
269 /* first, check if it's an exported symbol */
271 sym
= find_symbol(name
, NULL
, NULL
, true, true);
279 /* otherwise check if it's in another .o within the patch module */
280 return klp_find_object_symbol(pmod
->name
, name
, addr
);
283 static int klp_write_object_relocations(struct module
*pmod
,
284 struct klp_object
*obj
)
287 struct klp_reloc
*reloc
;
289 if (WARN_ON(!klp_is_object_loaded(obj
)))
292 if (WARN_ON(!obj
->relocs
))
295 for (reloc
= obj
->relocs
; reloc
->name
; reloc
++) {
296 if (!klp_is_module(obj
)) {
297 ret
= klp_verify_vmlinux_symbol(reloc
->name
,
302 /* module, reloc->val needs to be discovered */
304 ret
= klp_find_external_symbol(pmod
,
308 ret
= klp_find_object_symbol(obj
->mod
->name
,
314 ret
= klp_write_module_reloc(pmod
, reloc
->type
, reloc
->loc
,
315 reloc
->val
+ reloc
->addend
);
317 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
318 reloc
->name
, reloc
->val
, ret
);
326 static void notrace
klp_ftrace_handler(unsigned long ip
,
327 unsigned long parent_ip
,
328 struct ftrace_ops
*fops
,
329 struct pt_regs
*regs
)
332 struct klp_func
*func
;
334 ops
= container_of(fops
, struct klp_ops
, fops
);
337 func
= list_first_or_null_rcu(&ops
->func_stack
, struct klp_func
,
339 if (WARN_ON_ONCE(!func
))
342 klp_arch_set_pc(regs
, (unsigned long)func
->new_func
);
347 static void klp_disable_func(struct klp_func
*func
)
351 if (WARN_ON(func
->state
!= KLP_ENABLED
))
353 if (WARN_ON(!func
->old_addr
))
356 ops
= klp_find_ops(func
->old_addr
);
360 if (list_is_singular(&ops
->func_stack
)) {
361 WARN_ON(unregister_ftrace_function(&ops
->fops
));
362 WARN_ON(ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0));
364 list_del_rcu(&func
->stack_node
);
365 list_del(&ops
->node
);
368 list_del_rcu(&func
->stack_node
);
371 func
->state
= KLP_DISABLED
;
374 static int klp_enable_func(struct klp_func
*func
)
379 if (WARN_ON(!func
->old_addr
))
382 if (WARN_ON(func
->state
!= KLP_DISABLED
))
385 ops
= klp_find_ops(func
->old_addr
);
387 ops
= kzalloc(sizeof(*ops
), GFP_KERNEL
);
391 ops
->fops
.func
= klp_ftrace_handler
;
392 ops
->fops
.flags
= FTRACE_OPS_FL_SAVE_REGS
|
393 FTRACE_OPS_FL_DYNAMIC
|
394 FTRACE_OPS_FL_IPMODIFY
;
396 list_add(&ops
->node
, &klp_ops
);
398 INIT_LIST_HEAD(&ops
->func_stack
);
399 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
401 ret
= ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 0, 0);
403 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
404 func
->old_name
, ret
);
408 ret
= register_ftrace_function(&ops
->fops
);
410 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
411 func
->old_name
, ret
);
412 ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0);
418 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
421 func
->state
= KLP_ENABLED
;
426 list_del_rcu(&func
->stack_node
);
427 list_del(&ops
->node
);
432 static void klp_disable_object(struct klp_object
*obj
)
434 struct klp_func
*func
;
436 klp_for_each_func(obj
, func
)
437 if (func
->state
== KLP_ENABLED
)
438 klp_disable_func(func
);
440 obj
->state
= KLP_DISABLED
;
443 static int klp_enable_object(struct klp_object
*obj
)
445 struct klp_func
*func
;
448 if (WARN_ON(obj
->state
!= KLP_DISABLED
))
451 if (WARN_ON(!klp_is_object_loaded(obj
)))
454 klp_for_each_func(obj
, func
) {
455 ret
= klp_enable_func(func
);
457 klp_disable_object(obj
);
461 obj
->state
= KLP_ENABLED
;
466 static int __klp_disable_patch(struct klp_patch
*patch
)
468 struct klp_object
*obj
;
470 /* enforce stacking: only the last enabled patch can be disabled */
471 if (!list_is_last(&patch
->list
, &klp_patches
) &&
472 list_next_entry(patch
, list
)->state
== KLP_ENABLED
)
475 pr_notice("disabling patch '%s'\n", patch
->mod
->name
);
477 klp_for_each_object(patch
, obj
) {
478 if (obj
->state
== KLP_ENABLED
)
479 klp_disable_object(obj
);
482 patch
->state
= KLP_DISABLED
;
488 * klp_disable_patch() - disables a registered patch
489 * @patch: The registered, enabled patch to be disabled
491 * Unregisters the patched functions from ftrace.
493 * Return: 0 on success, otherwise error
495 int klp_disable_patch(struct klp_patch
*patch
)
499 mutex_lock(&klp_mutex
);
501 if (!klp_is_patch_registered(patch
)) {
506 if (patch
->state
== KLP_DISABLED
) {
511 ret
= __klp_disable_patch(patch
);
514 mutex_unlock(&klp_mutex
);
517 EXPORT_SYMBOL_GPL(klp_disable_patch
);
519 static int __klp_enable_patch(struct klp_patch
*patch
)
521 struct klp_object
*obj
;
524 if (WARN_ON(patch
->state
!= KLP_DISABLED
))
527 /* enforce stacking: only the first disabled patch can be enabled */
528 if (patch
->list
.prev
!= &klp_patches
&&
529 list_prev_entry(patch
, list
)->state
== KLP_DISABLED
)
532 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
533 add_taint(TAINT_LIVEPATCH
, LOCKDEP_STILL_OK
);
535 pr_notice("enabling patch '%s'\n", patch
->mod
->name
);
537 klp_for_each_object(patch
, obj
) {
538 if (!klp_is_object_loaded(obj
))
541 ret
= klp_enable_object(obj
);
546 patch
->state
= KLP_ENABLED
;
551 WARN_ON(__klp_disable_patch(patch
));
556 * klp_enable_patch() - enables a registered patch
557 * @patch: The registered, disabled patch to be enabled
559 * Performs the needed symbol lookups and code relocations,
560 * then registers the patched functions with ftrace.
562 * Return: 0 on success, otherwise error
564 int klp_enable_patch(struct klp_patch
*patch
)
568 mutex_lock(&klp_mutex
);
570 if (!klp_is_patch_registered(patch
)) {
575 ret
= __klp_enable_patch(patch
);
578 mutex_unlock(&klp_mutex
);
581 EXPORT_SYMBOL_GPL(klp_enable_patch
);
586 * /sys/kernel/livepatch
587 * /sys/kernel/livepatch/<patch>
588 * /sys/kernel/livepatch/<patch>/enabled
589 * /sys/kernel/livepatch/<patch>/<object>
590 * /sys/kernel/livepatch/<patch>/<object>/<func>
593 static ssize_t
enabled_store(struct kobject
*kobj
, struct kobj_attribute
*attr
,
594 const char *buf
, size_t count
)
596 struct klp_patch
*patch
;
600 ret
= kstrtoul(buf
, 10, &val
);
604 if (val
!= KLP_DISABLED
&& val
!= KLP_ENABLED
)
607 patch
= container_of(kobj
, struct klp_patch
, kobj
);
609 mutex_lock(&klp_mutex
);
611 if (val
== patch
->state
) {
612 /* already in requested state */
617 if (val
== KLP_ENABLED
) {
618 ret
= __klp_enable_patch(patch
);
622 ret
= __klp_disable_patch(patch
);
627 mutex_unlock(&klp_mutex
);
632 mutex_unlock(&klp_mutex
);
636 static ssize_t
enabled_show(struct kobject
*kobj
,
637 struct kobj_attribute
*attr
, char *buf
)
639 struct klp_patch
*patch
;
641 patch
= container_of(kobj
, struct klp_patch
, kobj
);
642 return snprintf(buf
, PAGE_SIZE
-1, "%d\n", patch
->state
);
645 static struct kobj_attribute enabled_kobj_attr
= __ATTR_RW(enabled
);
646 static struct attribute
*klp_patch_attrs
[] = {
647 &enabled_kobj_attr
.attr
,
651 static void klp_kobj_release_patch(struct kobject
*kobj
)
654 * Once we have a consistency model we'll need to module_put() the
655 * patch module here. See klp_register_patch() for more details.
659 static struct kobj_type klp_ktype_patch
= {
660 .release
= klp_kobj_release_patch
,
661 .sysfs_ops
= &kobj_sysfs_ops
,
662 .default_attrs
= klp_patch_attrs
,
665 static void klp_kobj_release_object(struct kobject
*kobj
)
669 static struct kobj_type klp_ktype_object
= {
670 .release
= klp_kobj_release_object
,
671 .sysfs_ops
= &kobj_sysfs_ops
,
674 static void klp_kobj_release_func(struct kobject
*kobj
)
678 static struct kobj_type klp_ktype_func
= {
679 .release
= klp_kobj_release_func
,
680 .sysfs_ops
= &kobj_sysfs_ops
,
684 * Free all functions' kobjects in the array up to some limit. When limit is
685 * NULL, all kobjects are freed.
687 static void klp_free_funcs_limited(struct klp_object
*obj
,
688 struct klp_func
*limit
)
690 struct klp_func
*func
;
692 for (func
= obj
->funcs
; func
->old_name
&& func
!= limit
; func
++)
693 kobject_put(&func
->kobj
);
696 /* Clean up when a patched object is unloaded */
697 static void klp_free_object_loaded(struct klp_object
*obj
)
699 struct klp_func
*func
;
703 klp_for_each_func(obj
, func
)
708 * Free all objects' kobjects in the array up to some limit. When limit is
709 * NULL, all kobjects are freed.
711 static void klp_free_objects_limited(struct klp_patch
*patch
,
712 struct klp_object
*limit
)
714 struct klp_object
*obj
;
716 for (obj
= patch
->objs
; obj
->funcs
&& obj
!= limit
; obj
++) {
717 klp_free_funcs_limited(obj
, NULL
);
718 kobject_put(&obj
->kobj
);
722 static void klp_free_patch(struct klp_patch
*patch
)
724 klp_free_objects_limited(patch
, NULL
);
725 if (!list_empty(&patch
->list
))
726 list_del(&patch
->list
);
727 kobject_put(&patch
->kobj
);
730 static int klp_init_func(struct klp_object
*obj
, struct klp_func
*func
)
732 INIT_LIST_HEAD(&func
->stack_node
);
733 func
->state
= KLP_DISABLED
;
735 return kobject_init_and_add(&func
->kobj
, &klp_ktype_func
,
736 &obj
->kobj
, "%s", func
->old_name
);
739 /* parts of the initialization that is done only when the object is loaded */
740 static int klp_init_object_loaded(struct klp_patch
*patch
,
741 struct klp_object
*obj
)
743 struct klp_func
*func
;
747 ret
= klp_write_object_relocations(patch
->mod
, obj
);
752 klp_for_each_func(obj
, func
) {
753 ret
= klp_find_verify_func_addr(obj
, func
);
761 static int klp_init_object(struct klp_patch
*patch
, struct klp_object
*obj
)
763 struct klp_func
*func
;
770 obj
->state
= KLP_DISABLED
;
773 klp_find_object_module(obj
);
775 name
= klp_is_module(obj
) ? obj
->name
: "vmlinux";
776 ret
= kobject_init_and_add(&obj
->kobj
, &klp_ktype_object
,
777 &patch
->kobj
, "%s", name
);
781 klp_for_each_func(obj
, func
) {
782 ret
= klp_init_func(obj
, func
);
787 if (klp_is_object_loaded(obj
)) {
788 ret
= klp_init_object_loaded(patch
, obj
);
796 klp_free_funcs_limited(obj
, func
);
797 kobject_put(&obj
->kobj
);
801 static int klp_init_patch(struct klp_patch
*patch
)
803 struct klp_object
*obj
;
809 mutex_lock(&klp_mutex
);
811 patch
->state
= KLP_DISABLED
;
813 ret
= kobject_init_and_add(&patch
->kobj
, &klp_ktype_patch
,
814 klp_root_kobj
, "%s", patch
->mod
->name
);
818 klp_for_each_object(patch
, obj
) {
819 ret
= klp_init_object(patch
, obj
);
824 list_add_tail(&patch
->list
, &klp_patches
);
826 mutex_unlock(&klp_mutex
);
831 klp_free_objects_limited(patch
, obj
);
832 kobject_put(&patch
->kobj
);
834 mutex_unlock(&klp_mutex
);
839 * klp_unregister_patch() - unregisters a patch
840 * @patch: Disabled patch to be unregistered
842 * Frees the data structures and removes the sysfs interface.
844 * Return: 0 on success, otherwise error
846 int klp_unregister_patch(struct klp_patch
*patch
)
850 mutex_lock(&klp_mutex
);
852 if (!klp_is_patch_registered(patch
)) {
857 if (patch
->state
== KLP_ENABLED
) {
862 klp_free_patch(patch
);
865 mutex_unlock(&klp_mutex
);
868 EXPORT_SYMBOL_GPL(klp_unregister_patch
);
871 * klp_register_patch() - registers a patch
872 * @patch: Patch to be registered
874 * Initializes the data structure associated with the patch and
875 * creates the sysfs interface.
877 * Return: 0 on success, otherwise error
879 int klp_register_patch(struct klp_patch
*patch
)
883 if (!klp_initialized())
886 if (!patch
|| !patch
->mod
)
890 * A reference is taken on the patch module to prevent it from being
891 * unloaded. Right now, we don't allow patch modules to unload since
892 * there is currently no method to determine if a thread is still
893 * running in the patched code contained in the patch module once
894 * the ftrace registration is successful.
896 if (!try_module_get(patch
->mod
))
899 ret
= klp_init_patch(patch
);
901 module_put(patch
->mod
);
905 EXPORT_SYMBOL_GPL(klp_register_patch
);
907 static int klp_module_notify_coming(struct klp_patch
*patch
,
908 struct klp_object
*obj
)
910 struct module
*pmod
= patch
->mod
;
911 struct module
*mod
= obj
->mod
;
914 ret
= klp_init_object_loaded(patch
, obj
);
916 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
917 pmod
->name
, mod
->name
, ret
);
921 if (patch
->state
== KLP_DISABLED
)
924 pr_notice("applying patch '%s' to loading module '%s'\n",
925 pmod
->name
, mod
->name
);
927 ret
= klp_enable_object(obj
);
929 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
930 pmod
->name
, mod
->name
, ret
);
934 static void klp_module_notify_going(struct klp_patch
*patch
,
935 struct klp_object
*obj
)
937 struct module
*pmod
= patch
->mod
;
938 struct module
*mod
= obj
->mod
;
940 if (patch
->state
== KLP_DISABLED
)
943 pr_notice("reverting patch '%s' on unloading module '%s'\n",
944 pmod
->name
, mod
->name
);
946 klp_disable_object(obj
);
949 klp_free_object_loaded(obj
);
952 static int klp_module_notify(struct notifier_block
*nb
, unsigned long action
,
956 struct module
*mod
= data
;
957 struct klp_patch
*patch
;
958 struct klp_object
*obj
;
960 if (action
!= MODULE_STATE_COMING
&& action
!= MODULE_STATE_GOING
)
963 mutex_lock(&klp_mutex
);
966 * Each module has to know that the notifier has been called.
967 * We never know what module will get patched by a new patch.
969 if (action
== MODULE_STATE_COMING
)
970 mod
->klp_alive
= true;
971 else /* MODULE_STATE_GOING */
972 mod
->klp_alive
= false;
974 list_for_each_entry(patch
, &klp_patches
, list
) {
975 klp_for_each_object(patch
, obj
) {
976 if (!klp_is_module(obj
) || strcmp(obj
->name
, mod
->name
))
979 if (action
== MODULE_STATE_COMING
) {
981 ret
= klp_module_notify_coming(patch
, obj
);
984 pr_warn("patch '%s' is in an inconsistent state!\n",
987 } else /* MODULE_STATE_GOING */
988 klp_module_notify_going(patch
, obj
);
994 mutex_unlock(&klp_mutex
);
999 static struct notifier_block klp_module_nb
= {
1000 .notifier_call
= klp_module_notify
,
1001 .priority
= INT_MIN
+1, /* called late but before ftrace notifier */
1004 static int __init
klp_init(void)
1008 ret
= klp_check_compiler_support();
1010 pr_info("Your compiler is too old; turning off.\n");
1014 ret
= register_module_notifier(&klp_module_nb
);
1018 klp_root_kobj
= kobject_create_and_add("livepatch", kernel_kobj
);
1019 if (!klp_root_kobj
) {
1027 unregister_module_notifier(&klp_module_nb
);
1031 module_init(klp_init
);