#include <linux/kallsyms.h>
#include <linux/livepatch.h>
-/*
- * The klp_mutex protects the klp_patches list and state transitions of any
- * structure reachable from the patches list. References to any structure must
- * be obtained under mutex protection.
+/**
+ * struct klp_ops - structure for tracking registered ftrace ops structs
+ *
+ * A single ftrace_ops is shared between all enabled replacement functions
+ * (klp_func structs) which have the same old_addr. This allows the switch
+ * between function versions to happen instantaneously by updating the klp_ops
+ * struct's func_stack list. The winner is the klp_func at the top of the
+ * func_stack (front of the list).
+ *
+ * @node: node for the global klp_ops list
+ * @func_stack: list head for the stack of klp_func's (active func is on top)
+ * @fops: registered ftrace ops struct
*/
+struct klp_ops {
+ struct list_head node;
+ struct list_head func_stack;
+ struct ftrace_ops fops;
+};
+/*
+ * The klp_mutex protects the global lists and state transitions of any
+ * structure reachable from them. References to any structure must be obtained
+ * under mutex protection (except in klp_ftrace_handler(), which uses RCU to
+ * ensure it gets consistent data).
+ */
static DEFINE_MUTEX(klp_mutex);
+
static LIST_HEAD(klp_patches);
+static LIST_HEAD(klp_ops);
static struct kobject *klp_root_kobj;
+static struct klp_ops *klp_find_ops(unsigned long old_addr)
+{
+ struct klp_ops *ops;
+ struct klp_func *func;
+
+ list_for_each_entry(ops, &klp_ops, node) {
+ func = list_first_entry(&ops->func_stack, struct klp_func,
+ stack_node);
+ if (func->old_addr == old_addr)
+ return ops;
+ }
+
+ return NULL;
+}
+
static bool klp_is_module(struct klp_object *obj)
{
return obj->name;
/* sets obj->mod if object is not vmlinux and module is found */
static void klp_find_object_module(struct klp_object *obj)
{
+ struct module *mod;
+
if (!klp_is_module(obj))
return;
mutex_lock(&module_mutex);
/*
- * We don't need to take a reference on the module here because we have
- * the klp_mutex, which is also taken by the module notifier. This
- * prevents any module from unloading until we release the klp_mutex.
+ * We do not want to block removal of patched modules and therefore
+ * we do not take a reference here. The patches are removed by
+ * a going module handler instead.
+ */
+ mod = find_module(obj->name);
+ /*
+ * Do not mess work of the module coming and going notifiers.
+ * Note that the patch might still be needed before the going handler
+ * is called. Module functions can be called even in the GOING state
+ * until mod->exit() finishes. This is especially important for
+ * patches that modify semantic of the functions.
*/
- obj->mod = find_module(obj->name);
+ if (mod && mod->klp_alive)
+ obj->mod = mod;
+
mutex_unlock(&module_mutex);
}
if (kallsyms_on_each_symbol(klp_verify_callback, &args))
return 0;
- pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?",
+ pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?\n",
name, addr);
return -EINVAL;
}
/* first, check if it's an exported symbol */
preempt_disable();
sym = find_symbol(name, NULL, NULL, true, true);
- preempt_enable();
if (sym) {
*addr = sym->value;
+ preempt_enable();
return 0;
}
+ preempt_enable();
/* otherwise check if it's in another .o within the patch module */
return klp_find_object_symbol(pmod->name, name, addr);
static void notrace klp_ftrace_handler(unsigned long ip,
unsigned long parent_ip,
- struct ftrace_ops *ops,
+ struct ftrace_ops *fops,
struct pt_regs *regs)
{
- struct klp_func *func = ops->private;
+ struct klp_ops *ops;
+ struct klp_func *func;
+
+ ops = container_of(fops, struct klp_ops, fops);
+
+ rcu_read_lock();
+ func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
+ stack_node);
+ if (WARN_ON_ONCE(!func))
+ goto unlock;
klp_arch_set_pc(regs, (unsigned long)func->new_func);
+unlock:
+ rcu_read_unlock();
}
static int klp_disable_func(struct klp_func *func)
{
+ struct klp_ops *ops;
int ret;
if (WARN_ON(func->state != KLP_ENABLED))
if (WARN_ON(!func->old_addr))
return -EINVAL;
- ret = unregister_ftrace_function(func->fops);
- if (ret) {
- pr_err("failed to unregister ftrace handler for function '%s' (%d)\n",
- func->old_name, ret);
- return ret;
- }
+ ops = klp_find_ops(func->old_addr);
+ if (WARN_ON(!ops))
+ return -EINVAL;
- ret = ftrace_set_filter_ip(func->fops, func->old_addr, 1, 0);
- if (ret)
- pr_warn("function unregister succeeded but failed to clear the filter\n");
+ if (list_is_singular(&ops->func_stack)) {
+ ret = unregister_ftrace_function(&ops->fops);
+ if (ret) {
+ pr_err("failed to unregister ftrace handler for function '%s' (%d)\n",
+ func->old_name, ret);
+ return ret;
+ }
+
+ ret = ftrace_set_filter_ip(&ops->fops, func->old_addr, 1, 0);
+ if (ret)
+ pr_warn("function unregister succeeded but failed to clear the filter\n");
+
+ list_del_rcu(&func->stack_node);
+ list_del(&ops->node);
+ kfree(ops);
+ } else {
+ list_del_rcu(&func->stack_node);
+ }
func->state = KLP_DISABLED;
static int klp_enable_func(struct klp_func *func)
{
+ struct klp_ops *ops;
int ret;
if (WARN_ON(!func->old_addr))
if (WARN_ON(func->state != KLP_DISABLED))
return -EINVAL;
- ret = ftrace_set_filter_ip(func->fops, func->old_addr, 0, 0);
- if (ret) {
- pr_err("failed to set ftrace filter for function '%s' (%d)\n",
- func->old_name, ret);
- return ret;
- }
+ ops = klp_find_ops(func->old_addr);
+ if (!ops) {
+ ops = kzalloc(sizeof(*ops), GFP_KERNEL);
+ if (!ops)
+ return -ENOMEM;
+
+ ops->fops.func = klp_ftrace_handler;
+ ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
+ FTRACE_OPS_FL_DYNAMIC |
+ FTRACE_OPS_FL_IPMODIFY;
+
+ list_add(&ops->node, &klp_ops);
+
+ INIT_LIST_HEAD(&ops->func_stack);
+ list_add_rcu(&func->stack_node, &ops->func_stack);
+
+ ret = ftrace_set_filter_ip(&ops->fops, func->old_addr, 0, 0);
+ if (ret) {
+ pr_err("failed to set ftrace filter for function '%s' (%d)\n",
+ func->old_name, ret);
+ goto err;
+ }
+
+ ret = register_ftrace_function(&ops->fops);
+ if (ret) {
+ pr_err("failed to register ftrace handler for function '%s' (%d)\n",
+ func->old_name, ret);
+ ftrace_set_filter_ip(&ops->fops, func->old_addr, 1, 0);
+ goto err;
+ }
+
- ret = register_ftrace_function(func->fops);
- if (ret) {
- pr_err("failed to register ftrace handler for function '%s' (%d)\n",
- func->old_name, ret);
- ftrace_set_filter_ip(func->fops, func->old_addr, 1, 0);
} else {
- func->state = KLP_ENABLED;
+ list_add_rcu(&func->stack_node, &ops->func_stack);
}
+ func->state = KLP_ENABLED;
+
+ return 0;
+
+err:
+ list_del_rcu(&func->stack_node);
+ list_del(&ops->node);
+ kfree(ops);
return ret;
}
static void klp_kobj_release_func(struct kobject *kobj)
{
- struct klp_func *func;
-
- func = container_of(kobj, struct klp_func, kobj);
- kfree(func->fops);
}
static struct kobj_type klp_ktype_func = {
static int klp_init_func(struct klp_object *obj, struct klp_func *func)
{
- struct ftrace_ops *ops;
- int ret;
-
- ops = kzalloc(sizeof(*ops), GFP_KERNEL);
- if (!ops)
- return -ENOMEM;
-
- ops->private = func;
- ops->func = klp_ftrace_handler;
- ops->flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_DYNAMIC |
- FTRACE_OPS_FL_IPMODIFY;
- func->fops = ops;
+ INIT_LIST_HEAD(&func->stack_node);
func->state = KLP_DISABLED;
- ret = kobject_init_and_add(&func->kobj, &klp_ktype_func,
- obj->kobj, func->old_name);
- if (ret) {
- kfree(func->fops);
- return ret;
- }
-
- return 0;
+ return kobject_init_and_add(&func->kobj, &klp_ktype_func,
+ obj->kobj, "%s", func->old_name);
}
/* parts of the initialization that is done only when the object is loaded */
return -EINVAL;
obj->state = KLP_DISABLED;
+ obj->mod = NULL;
klp_find_object_module(obj);
patch->state = KLP_DISABLED;
ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
- klp_root_kobj, patch->mod->name);
+ klp_root_kobj, "%s", patch->mod->name);
if (ret)
goto unlock;
mutex_lock(&klp_mutex);
+ /*
+ * Each module has to know that the notifier has been called.
+ * We never know what module will get patched by a new patch.
+ */
+ if (action == MODULE_STATE_COMING)
+ mod->klp_alive = true;
+ else /* MODULE_STATE_GOING */
+ mod->klp_alive = false;
+
list_for_each_entry(patch, &klp_patches, list) {
for (obj = patch->objs; obj->funcs; obj++) {
if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
projects / firefly-linux-kernel-4.4.55.git / blobdiff