2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * The klp_mutex protects the klp_patches list and state transitions of any
34 * structure reachable from the patches list. References to any structure must
35 * be obtained under mutex protection.
38 static DEFINE_MUTEX(klp_mutex);
39 static LIST_HEAD(klp_patches);
41 static struct kobject *klp_root_kobj;
43 static bool klp_is_module(struct klp_object *obj)
48 static bool klp_is_object_loaded(struct klp_object *obj)
50 return !obj->name || obj->mod;
53 /* sets obj->mod if object is not vmlinux and module is found */
54 static void klp_find_object_module(struct klp_object *obj)
56 if (!klp_is_module(obj))
59 mutex_lock(&module_mutex);
61 * We don't need to take a reference on the module here because we have
62 * the klp_mutex, which is also taken by the module notifier. This
63 * prevents any module from unloading until we release the klp_mutex.
65 obj->mod = find_module(obj->name);
66 mutex_unlock(&module_mutex);
69 /* klp_mutex must be held by caller */
70 static bool klp_is_patch_registered(struct klp_patch *patch)
72 struct klp_patch *mypatch;
74 list_for_each_entry(mypatch, &klp_patches, list)
81 static bool klp_initialized(void)
91 * If count == 0, the symbol was not found. If count == 1, a unique
92 * match was found and addr is set. If count > 1, there is
93 * unresolvable ambiguity among "count" number of symbols with the same
94 * name in the same object.
99 static int klp_find_callback(void *data, const char *name,
100 struct module *mod, unsigned long addr)
102 struct klp_find_arg *args = data;
104 if ((mod && !args->objname) || (!mod && args->objname))
107 if (strcmp(args->name, name))
110 if (args->objname && strcmp(args->objname, mod->name))
114 * args->addr might be overwritten if another match is found
115 * but klp_find_object_symbol() handles this and only returns the
116 * addr if count == 1.
124 static int klp_find_object_symbol(const char *objname, const char *name,
127 struct klp_find_arg args = {
134 kallsyms_on_each_symbol(klp_find_callback, &args);
137 pr_err("symbol '%s' not found in symbol table\n", name);
138 else if (args.count > 1)
139 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
140 args.count, name, objname);
150 struct klp_verify_args {
152 const unsigned long addr;
155 static int klp_verify_callback(void *data, const char *name,
156 struct module *mod, unsigned long addr)
158 struct klp_verify_args *args = data;
161 !strcmp(args->name, name) &&
168 static int klp_verify_vmlinux_symbol(const char *name, unsigned long addr)
170 struct klp_verify_args args = {
175 if (kallsyms_on_each_symbol(klp_verify_callback, &args))
178 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?",
183 static int klp_find_verify_func_addr(struct klp_object *obj,
184 struct klp_func *func)
188 #if defined(CONFIG_RANDOMIZE_BASE)
189 /* KASLR is enabled, disregard old_addr from user */
193 if (!func->old_addr || klp_is_module(obj))
194 ret = klp_find_object_symbol(obj->name, func->old_name,
197 ret = klp_verify_vmlinux_symbol(func->old_name,
204 * external symbols are located outside the parent object (where the parent
205 * object is either vmlinux or the kmod being patched).
207 static int klp_find_external_symbol(struct module *pmod, const char *name,
210 const struct kernel_symbol *sym;
212 /* first, check if it's an exported symbol */
214 sym = find_symbol(name, NULL, NULL, true, true);
221 /* otherwise check if it's in another .o within the patch module */
222 return klp_find_object_symbol(pmod->name, name, addr);
225 static int klp_write_object_relocations(struct module *pmod,
226 struct klp_object *obj)
229 struct klp_reloc *reloc;
231 if (WARN_ON(!klp_is_object_loaded(obj)))
234 if (WARN_ON(!obj->relocs))
237 for (reloc = obj->relocs; reloc->name; reloc++) {
238 if (!klp_is_module(obj)) {
239 ret = klp_verify_vmlinux_symbol(reloc->name,
244 /* module, reloc->val needs to be discovered */
246 ret = klp_find_external_symbol(pmod,
250 ret = klp_find_object_symbol(obj->mod->name,
256 ret = klp_write_module_reloc(pmod, reloc->type, reloc->loc,
257 reloc->val + reloc->addend);
259 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
260 reloc->name, reloc->val, ret);
268 static void notrace klp_ftrace_handler(unsigned long ip,
269 unsigned long parent_ip,
270 struct ftrace_ops *ops,
271 struct pt_regs *regs)
273 struct klp_func *func = ops->private;
275 klp_arch_set_pc(regs, (unsigned long)func->new_func);
278 static int klp_disable_func(struct klp_func *func)
282 if (WARN_ON(func->state != KLP_ENABLED))
285 if (WARN_ON(!func->old_addr))
288 ret = unregister_ftrace_function(func->fops);
290 pr_err("failed to unregister ftrace handler for function '%s' (%d)\n",
291 func->old_name, ret);
295 ret = ftrace_set_filter_ip(func->fops, func->old_addr, 1, 0);
297 pr_warn("function unregister succeeded but failed to clear the filter\n");
299 func->state = KLP_DISABLED;
304 static int klp_enable_func(struct klp_func *func)
308 if (WARN_ON(!func->old_addr))
311 if (WARN_ON(func->state != KLP_DISABLED))
314 ret = ftrace_set_filter_ip(func->fops, func->old_addr, 0, 0);
316 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
317 func->old_name, ret);
321 ret = register_ftrace_function(func->fops);
323 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
324 func->old_name, ret);
325 ftrace_set_filter_ip(func->fops, func->old_addr, 1, 0);
327 func->state = KLP_ENABLED;
333 static int klp_disable_object(struct klp_object *obj)
335 struct klp_func *func;
338 for (func = obj->funcs; func->old_name; func++) {
339 if (func->state != KLP_ENABLED)
342 ret = klp_disable_func(func);
347 obj->state = KLP_DISABLED;
352 static int klp_enable_object(struct klp_object *obj)
354 struct klp_func *func;
357 if (WARN_ON(obj->state != KLP_DISABLED))
360 if (WARN_ON(!klp_is_object_loaded(obj)))
363 for (func = obj->funcs; func->old_name; func++) {
364 ret = klp_enable_func(func);
368 obj->state = KLP_ENABLED;
373 WARN_ON(klp_disable_object(obj));
377 static int __klp_disable_patch(struct klp_patch *patch)
379 struct klp_object *obj;
382 pr_notice("disabling patch '%s'\n", patch->mod->name);
384 for (obj = patch->objs; obj->funcs; obj++) {
385 if (obj->state != KLP_ENABLED)
388 ret = klp_disable_object(obj);
393 patch->state = KLP_DISABLED;
399 * klp_disable_patch() - disables a registered patch
400 * @patch: The registered, enabled patch to be disabled
402 * Unregisters the patched functions from ftrace.
404 * Return: 0 on success, otherwise error
406 int klp_disable_patch(struct klp_patch *patch)
410 mutex_lock(&klp_mutex);
412 if (!klp_is_patch_registered(patch)) {
417 if (patch->state == KLP_DISABLED) {
422 ret = __klp_disable_patch(patch);
425 mutex_unlock(&klp_mutex);
428 EXPORT_SYMBOL_GPL(klp_disable_patch);
430 static int __klp_enable_patch(struct klp_patch *patch)
432 struct klp_object *obj;
435 if (WARN_ON(patch->state != KLP_DISABLED))
438 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
439 add_taint(TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
441 pr_notice("enabling patch '%s'\n", patch->mod->name);
443 for (obj = patch->objs; obj->funcs; obj++) {
444 klp_find_object_module(obj);
446 if (!klp_is_object_loaded(obj))
449 ret = klp_enable_object(obj);
454 patch->state = KLP_ENABLED;
459 WARN_ON(__klp_disable_patch(patch));
464 * klp_enable_patch() - enables a registered patch
465 * @patch: The registered, disabled patch to be enabled
467 * Performs the needed symbol lookups and code relocations,
468 * then registers the patched functions with ftrace.
470 * Return: 0 on success, otherwise error
472 int klp_enable_patch(struct klp_patch *patch)
476 mutex_lock(&klp_mutex);
478 if (!klp_is_patch_registered(patch)) {
483 ret = __klp_enable_patch(patch);
486 mutex_unlock(&klp_mutex);
489 EXPORT_SYMBOL_GPL(klp_enable_patch);
494 * /sys/kernel/livepatch
495 * /sys/kernel/livepatch/<patch>
496 * /sys/kernel/livepatch/<patch>/enabled
497 * /sys/kernel/livepatch/<patch>/<object>
498 * /sys/kernel/livepatch/<patch>/<object>/<func>
501 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
502 const char *buf, size_t count)
504 struct klp_patch *patch;
508 ret = kstrtoul(buf, 10, &val);
512 if (val != KLP_DISABLED && val != KLP_ENABLED)
515 patch = container_of(kobj, struct klp_patch, kobj);
517 mutex_lock(&klp_mutex);
519 if (val == patch->state) {
520 /* already in requested state */
525 if (val == KLP_ENABLED) {
526 ret = __klp_enable_patch(patch);
530 ret = __klp_disable_patch(patch);
535 mutex_unlock(&klp_mutex);
540 mutex_unlock(&klp_mutex);
544 static ssize_t enabled_show(struct kobject *kobj,
545 struct kobj_attribute *attr, char *buf)
547 struct klp_patch *patch;
549 patch = container_of(kobj, struct klp_patch, kobj);
550 return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->state);
553 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
554 static struct attribute *klp_patch_attrs[] = {
555 &enabled_kobj_attr.attr,
559 static void klp_kobj_release_patch(struct kobject *kobj)
562 * Once we have a consistency model we'll need to module_put() the
563 * patch module here. See klp_register_patch() for more details.
567 static struct kobj_type klp_ktype_patch = {
568 .release = klp_kobj_release_patch,
569 .sysfs_ops = &kobj_sysfs_ops,
570 .default_attrs = klp_patch_attrs,
573 static void klp_kobj_release_func(struct kobject *kobj)
575 struct klp_func *func;
577 func = container_of(kobj, struct klp_func, kobj);
581 static struct kobj_type klp_ktype_func = {
582 .release = klp_kobj_release_func,
583 .sysfs_ops = &kobj_sysfs_ops,
587 * Free all functions' kobjects in the array up to some limit. When limit is
588 * NULL, all kobjects are freed.
590 static void klp_free_funcs_limited(struct klp_object *obj,
591 struct klp_func *limit)
593 struct klp_func *func;
595 for (func = obj->funcs; func->old_name && func != limit; func++)
596 kobject_put(&func->kobj);
599 /* Clean up when a patched object is unloaded */
600 static void klp_free_object_loaded(struct klp_object *obj)
602 struct klp_func *func;
606 for (func = obj->funcs; func->old_name; func++)
611 * Free all objects' kobjects in the array up to some limit. When limit is
612 * NULL, all kobjects are freed.
614 static void klp_free_objects_limited(struct klp_patch *patch,
615 struct klp_object *limit)
617 struct klp_object *obj;
619 for (obj = patch->objs; obj->funcs && obj != limit; obj++) {
620 klp_free_funcs_limited(obj, NULL);
621 kobject_put(obj->kobj);
625 static void klp_free_patch(struct klp_patch *patch)
627 klp_free_objects_limited(patch, NULL);
628 if (!list_empty(&patch->list))
629 list_del(&patch->list);
630 kobject_put(&patch->kobj);
633 static int klp_init_func(struct klp_object *obj, struct klp_func *func)
635 struct ftrace_ops *ops;
638 ops = kzalloc(sizeof(*ops), GFP_KERNEL);
643 ops->func = klp_ftrace_handler;
644 ops->flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_DYNAMIC |
645 FTRACE_OPS_FL_IPMODIFY;
647 func->state = KLP_DISABLED;
649 ret = kobject_init_and_add(&func->kobj, &klp_ktype_func,
650 obj->kobj, func->old_name);
659 /* parts of the initialization that is done only when the object is loaded */
660 static int klp_init_object_loaded(struct klp_patch *patch,
661 struct klp_object *obj)
663 struct klp_func *func;
667 ret = klp_write_object_relocations(patch->mod, obj);
672 for (func = obj->funcs; func->old_name; func++) {
673 ret = klp_find_verify_func_addr(obj, func);
681 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
683 struct klp_func *func;
690 obj->state = KLP_DISABLED;
692 klp_find_object_module(obj);
694 name = klp_is_module(obj) ? obj->name : "vmlinux";
695 obj->kobj = kobject_create_and_add(name, &patch->kobj);
699 for (func = obj->funcs; func->old_name; func++) {
700 ret = klp_init_func(obj, func);
705 if (klp_is_object_loaded(obj)) {
706 ret = klp_init_object_loaded(patch, obj);
714 klp_free_funcs_limited(obj, func);
715 kobject_put(obj->kobj);
719 static int klp_init_patch(struct klp_patch *patch)
721 struct klp_object *obj;
727 mutex_lock(&klp_mutex);
729 patch->state = KLP_DISABLED;
731 ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
732 klp_root_kobj, patch->mod->name);
736 for (obj = patch->objs; obj->funcs; obj++) {
737 ret = klp_init_object(patch, obj);
742 list_add_tail(&patch->list, &klp_patches);
744 mutex_unlock(&klp_mutex);
749 klp_free_objects_limited(patch, obj);
750 kobject_put(&patch->kobj);
752 mutex_unlock(&klp_mutex);
757 * klp_unregister_patch() - unregisters a patch
758 * @patch: Disabled patch to be unregistered
760 * Frees the data structures and removes the sysfs interface.
762 * Return: 0 on success, otherwise error
764 int klp_unregister_patch(struct klp_patch *patch)
768 mutex_lock(&klp_mutex);
770 if (!klp_is_patch_registered(patch)) {
775 if (patch->state == KLP_ENABLED) {
780 klp_free_patch(patch);
783 mutex_unlock(&klp_mutex);
786 EXPORT_SYMBOL_GPL(klp_unregister_patch);
789 * klp_register_patch() - registers a patch
790 * @patch: Patch to be registered
792 * Initializes the data structure associated with the patch and
793 * creates the sysfs interface.
795 * Return: 0 on success, otherwise error
797 int klp_register_patch(struct klp_patch *patch)
801 if (!klp_initialized())
804 if (!patch || !patch->mod)
808 * A reference is taken on the patch module to prevent it from being
809 * unloaded. Right now, we don't allow patch modules to unload since
810 * there is currently no method to determine if a thread is still
811 * running in the patched code contained in the patch module once
812 * the ftrace registration is successful.
814 if (!try_module_get(patch->mod))
817 ret = klp_init_patch(patch);
819 module_put(patch->mod);
823 EXPORT_SYMBOL_GPL(klp_register_patch);
825 static void klp_module_notify_coming(struct klp_patch *patch,
826 struct klp_object *obj)
828 struct module *pmod = patch->mod;
829 struct module *mod = obj->mod;
832 ret = klp_init_object_loaded(patch, obj);
836 if (patch->state == KLP_DISABLED)
839 pr_notice("applying patch '%s' to loading module '%s'\n",
840 pmod->name, mod->name);
842 ret = klp_enable_object(obj);
847 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
848 pmod->name, mod->name, ret);
851 static void klp_module_notify_going(struct klp_patch *patch,
852 struct klp_object *obj)
854 struct module *pmod = patch->mod;
855 struct module *mod = obj->mod;
858 if (patch->state == KLP_DISABLED)
861 pr_notice("reverting patch '%s' on unloading module '%s'\n",
862 pmod->name, mod->name);
864 ret = klp_disable_object(obj);
866 pr_warn("failed to revert patch '%s' on module '%s' (%d)\n",
867 pmod->name, mod->name, ret);
870 klp_free_object_loaded(obj);
873 static int klp_module_notify(struct notifier_block *nb, unsigned long action,
876 struct module *mod = data;
877 struct klp_patch *patch;
878 struct klp_object *obj;
880 if (action != MODULE_STATE_COMING && action != MODULE_STATE_GOING)
883 mutex_lock(&klp_mutex);
885 list_for_each_entry(patch, &klp_patches, list) {
886 for (obj = patch->objs; obj->funcs; obj++) {
887 if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
890 if (action == MODULE_STATE_COMING) {
892 klp_module_notify_coming(patch, obj);
893 } else /* MODULE_STATE_GOING */
894 klp_module_notify_going(patch, obj);
900 mutex_unlock(&klp_mutex);
905 static struct notifier_block klp_module_nb = {
906 .notifier_call = klp_module_notify,
907 .priority = INT_MIN+1, /* called late but before ftrace notifier */
910 static int klp_init(void)
914 ret = klp_check_compiler_support();
916 pr_info("Your compiler is too old; turning off.\n");
920 ret = register_module_notifier(&klp_module_nb);
924 klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
925 if (!klp_root_kobj) {
933 unregister_module_notifier(&klp_module_nb);
937 module_init(klp_init);