projects
/
firefly-linux-kernel-4.4.55.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
ftrace: Use the rcu _notrace variants for rcu_dereference_raw() and friends
[firefly-linux-kernel-4.4.55.git]
/
kernel
/
trace
/
ftrace.c
diff --git
a/kernel/trace/ftrace.c
b/kernel/trace/ftrace.c
index 8a5c017bb50c141bfca4d8206ac2a1805d224185..6c508ff33c6206df8e028e1eab43e913565f927e 100644
(file)
--- a/
kernel/trace/ftrace.c
+++ b/
kernel/trace/ftrace.c
@@
-64,6
+64,13
@@
#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
+#ifdef CONFIG_DYNAMIC_FTRACE
+#define INIT_REGEX_LOCK(opsname) \
+ .regex_lock = __MUTEX_INITIALIZER(opsname.regex_lock),
+#else
+#define INIT_REGEX_LOCK(opsname)
+#endif
+
static struct ftrace_ops ftrace_list_end __read_mostly = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
static struct ftrace_ops ftrace_list_end __read_mostly = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
@@
-113,24
+120,34
@@
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
/*
* Traverse the ftrace_global_list, invoking all entries. The reason that we
/*
* Traverse the ftrace_global_list, invoking all entries. The reason that we
- * can use rcu_dereference_raw() is that elements removed from this list
+ * can use rcu_dereference_raw
_notrace
() is that elements removed from this list
* are simply leaked, so there is no need to interact with a grace-period
* are simply leaked, so there is no need to interact with a grace-period
- * mechanism. The rcu_dereference_raw() calls are needed to handle
+ * mechanism. The rcu_dereference_raw
_notrace
() calls are needed to handle
* concurrent insertions into the ftrace_global_list.
*
* Silly Alpha and silly pointer-speculation compiler optimizations!
*/
#define do_for_each_ftrace_op(op, list) \
* concurrent insertions into the ftrace_global_list.
*
* Silly Alpha and silly pointer-speculation compiler optimizations!
*/
#define do_for_each_ftrace_op(op, list) \
- op = rcu_dereference_raw(list); \
+ op = rcu_dereference_raw
_notrace
(list); \
do
/*
* Optimized for just a single item in the list (as that is the normal case).
*/
#define while_for_each_ftrace_op(op) \
do
/*
* Optimized for just a single item in the list (as that is the normal case).
*/
#define while_for_each_ftrace_op(op) \
- while (likely(op = rcu_dereference_raw((op)->next)) && \
+ while (likely(op = rcu_dereference_raw
_notrace
((op)->next)) && \
unlikely((op) != &ftrace_list_end))
unlikely((op) != &ftrace_list_end))
+static inline void ftrace_ops_init(struct ftrace_ops *ops)
+{
+#ifdef CONFIG_DYNAMIC_FTRACE
+ if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
+ mutex_init(&ops->regex_lock);
+ ops->flags |= FTRACE_OPS_FL_INITIALIZED;
+ }
+#endif
+}
+
/**
* ftrace_nr_registered_ops - return number of ops registered
*
/**
* ftrace_nr_registered_ops - return number of ops registered
*
@@
-762,7
+779,7
@@
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
if (hlist_empty(hhd))
return NULL;
if (hlist_empty(hhd))
return NULL;
- hlist_for_each_entry_rcu(rec, hhd, node) {
+ hlist_for_each_entry_rcu
_notrace
(rec, hhd, node) {
if (rec->ip == ip)
return rec;
}
if (rec->ip == ip)
return rec;
}
@@
-907,7
+924,8
@@
static void unregister_ftrace_profiler(void)
#else
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
.func = function_profile_call,
#else
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
.func = function_profile_call,
- .flags = FTRACE_OPS_FL_RECURSION_SAFE,
+ .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
+ INIT_REGEX_LOCK(ftrace_profile_ops)
};
static int register_ftrace_profiler(void)
};
static int register_ftrace_profiler(void)
@@
-1103,11
+1121,10
@@
static struct ftrace_ops global_ops = {
.func = ftrace_stub,
.notrace_hash = EMPTY_HASH,
.filter_hash = EMPTY_HASH,
.func = ftrace_stub,
.notrace_hash = EMPTY_HASH,
.filter_hash = EMPTY_HASH,
- .flags = FTRACE_OPS_FL_RECURSION_SAFE,
+ .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
+ INIT_REGEX_LOCK(global_ops)
};
};
-static DEFINE_MUTEX(ftrace_regex_lock);
-
struct ftrace_page {
struct ftrace_page *next;
struct dyn_ftrace *records;
struct ftrace_page {
struct ftrace_page *next;
struct dyn_ftrace *records;
@@
-1148,7
+1165,7
@@
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
hhd = &hash->buckets[key];
hhd = &hash->buckets[key];
- hlist_for_each_entry_rcu(entry, hhd, hlist) {
+ hlist_for_each_entry_rcu
_notrace
(entry, hhd, hlist) {
if (entry->ip == ip)
return entry;
}
if (entry->ip == ip)
return entry;
}
@@
-1247,6
+1264,7
@@
static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
void ftrace_free_filter(struct ftrace_ops *ops)
{
void ftrace_free_filter(struct ftrace_ops *ops)
{
+ ftrace_ops_init(ops);
free_ftrace_hash(ops->filter_hash);
free_ftrace_hash(ops->notrace_hash);
}
free_ftrace_hash(ops->filter_hash);
free_ftrace_hash(ops->notrace_hash);
}
@@
-1404,8
+1422,8
@@
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
struct ftrace_hash *notrace_hash;
int ret;
struct ftrace_hash *notrace_hash;
int ret;
- filter_hash = rcu_dereference_raw(ops->filter_hash);
- notrace_hash = rcu_dereference_raw(ops->notrace_hash);
+ filter_hash = rcu_dereference_raw
_notrace
(ops->filter_hash);
+ notrace_hash = rcu_dereference_raw
_notrace
(ops->notrace_hash);
if ((ftrace_hash_empty(filter_hash) ||
ftrace_lookup_ip(filter_hash, ip)) &&
if ((ftrace_hash_empty(filter_hash) ||
ftrace_lookup_ip(filter_hash, ip)) &&
@@
-2441,7
+2459,7
@@
t_next(struct seq_file *m, void *v, loff_t *pos)
!ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
((iter->flags & FTRACE_ITER_ENABLED) &&
!ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
((iter->flags & FTRACE_ITER_ENABLED) &&
- !(rec->flags &
~FTRACE_FL_MASK
))) {
+ !(rec->flags &
FTRACE_FL_ENABLED
))) {
rec = NULL;
goto retry;
rec = NULL;
goto retry;
@@
-2624,6
+2642,8
@@
ftrace_regex_open(struct ftrace_ops *ops, int flag,
struct ftrace_hash *hash;
int ret = 0;
struct ftrace_hash *hash;
int ret = 0;
+ ftrace_ops_init(ops);
+
if (unlikely(ftrace_disabled))
return -ENODEV;
if (unlikely(ftrace_disabled))
return -ENODEV;
@@
-2636,28
+2656,26
@@
ftrace_regex_open(struct ftrace_ops *ops, int flag,
return -ENOMEM;
}
return -ENOMEM;
}
+ iter->ops = ops;
+ iter->flags = flag;
+
+ mutex_lock(&ops->regex_lock);
+
if (flag & FTRACE_ITER_NOTRACE)
hash = ops->notrace_hash;
else
hash = ops->filter_hash;
if (flag & FTRACE_ITER_NOTRACE)
hash = ops->notrace_hash;
else
hash = ops->filter_hash;
- iter->ops = ops;
- iter->flags = flag;
-
if (file->f_mode & FMODE_WRITE) {
if (file->f_mode & FMODE_WRITE) {
- mutex_lock(&ftrace_lock);
iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
- mutex_unlock(&ftrace_lock);
-
if (!iter->hash) {
trace_parser_put(&iter->parser);
kfree(iter);
if (!iter->hash) {
trace_parser_put(&iter->parser);
kfree(iter);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto out_unlock;
}
}
}
}
- mutex_lock(&ftrace_regex_lock);
-
if ((file->f_mode & FMODE_WRITE) &&
(file->f_flags & O_TRUNC))
ftrace_filter_reset(iter->hash);
if ((file->f_mode & FMODE_WRITE) &&
(file->f_flags & O_TRUNC))
ftrace_filter_reset(iter->hash);
@@
-2677,7
+2695,9
@@
ftrace_regex_open(struct ftrace_ops *ops, int flag,
}
} else
file->private_data = iter;
}
} else
file->private_data = iter;
- mutex_unlock(&ftrace_regex_lock);
+
+ out_unlock:
+ mutex_unlock(&ops->regex_lock);
return ret;
}
return ret;
}
@@
-2900,7
+2920,7
@@
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
* on the hash. rcu_read_lock is too dangerous here.
*/
preempt_disable_notrace();
* on the hash. rcu_read_lock is too dangerous here.
*/
preempt_disable_notrace();
- hlist_for_each_entry_rcu(entry, hhd, node) {
+ hlist_for_each_entry_rcu
_notrace
(entry, hhd, node) {
if (entry->ip == ip)
entry->ops->func(ip, parent_ip, &entry->data);
}
if (entry->ip == ip)
entry->ops->func(ip, parent_ip, &entry->data);
}
@@
-2910,6
+2930,8
@@
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
static struct ftrace_ops trace_probe_ops __read_mostly =
{
.func = function_trace_probe_call,
static struct ftrace_ops trace_probe_ops __read_mostly =
{
.func = function_trace_probe_call,
+ .flags = FTRACE_OPS_FL_INITIALIZED,
+ INIT_REGEX_LOCK(trace_probe_ops)
};
static int ftrace_probe_registered;
};
static int ftrace_probe_registered;
@@
-2919,8
+2941,12
@@
static void __enable_ftrace_function_probe(void)
int ret;
int i;
int ret;
int i;
- if (ftrace_probe_registered)
+ if (ftrace_probe_registered) {
+ /* still need to update the function call sites */
+ if (ftrace_enabled)
+ ftrace_run_update_code(FTRACE_UPDATE_CALLS);
return;
return;
+ }
for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
struct hlist_head *hhd = &ftrace_func_hash[i];
for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
struct hlist_head *hhd = &ftrace_func_hash[i];
@@
-2990,19
+3016,21
@@
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
if (WARN_ON(not))
return -EINVAL;
if (WARN_ON(not))
return -EINVAL;
- mutex_lock(&
ftrace
_lock);
+ mutex_lock(&
trace_probe_ops.regex
_lock);
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
if (!hash) {
count = -ENOMEM;
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
if (!hash) {
count = -ENOMEM;
- goto out
_unlock
;
+ goto out;
}
if (unlikely(ftrace_disabled)) {
count = -ENODEV;
}
if (unlikely(ftrace_disabled)) {
count = -ENODEV;
- goto out
_unlock
;
+ goto out;
}
}
+ mutex_lock(&ftrace_lock);
+
do_for_each_ftrace_rec(pg, rec) {
if (!ftrace_match_record(rec, NULL, search, len, type))
do_for_each_ftrace_rec(pg, rec) {
if (!ftrace_match_record(rec, NULL, search, len, type))
@@
-3056,6
+3084,8
@@
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
out_unlock:
mutex_unlock(&ftrace_lock);
out_unlock:
mutex_unlock(&ftrace_lock);
+ out:
+ mutex_unlock(&trace_probe_ops.regex_lock);
free_ftrace_hash(hash);
return count;
free_ftrace_hash(hash);
return count;
@@
-3095,7
+3125,7
@@
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
return;
}
return;
}
- mutex_lock(&
ftrace
_lock);
+ mutex_lock(&
trace_probe_ops.regex
_lock);
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
if (!hash)
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
if (!hash)
@@
-3133,6
+3163,7
@@
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
list_add(&entry->free_list, &free_list);
}
}
list_add(&entry->free_list, &free_list);
}
}
+ mutex_lock(&ftrace_lock);
__disable_ftrace_function_probe();
/*
* Remove after the disable is called. Otherwise, if the last
__disable_ftrace_function_probe();
/*
* Remove after the disable is called. Otherwise, if the last
@@
-3144,9
+3175,10
@@
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
list_del(&entry->free_list);
ftrace_free_entry(entry);
}
list_del(&entry->free_list);
ftrace_free_entry(entry);
}
+ mutex_unlock(&ftrace_lock);
out_unlock:
out_unlock:
- mutex_unlock(&
ftrace
_lock);
+ mutex_unlock(&
trace_probe_ops.regex
_lock);
free_ftrace_hash(hash);
}
free_ftrace_hash(hash);
}
@@
-3256,18
+3288,17
@@
ftrace_regex_write(struct file *file, const char __user *ubuf,
if (!cnt)
return 0;
if (!cnt)
return 0;
- mutex_lock(&ftrace_regex_lock);
-
- ret = -ENODEV;
- if (unlikely(ftrace_disabled))
- goto out_unlock;
-
if (file->f_mode & FMODE_READ) {
struct seq_file *m = file->private_data;
iter = m->private;
} else
iter = file->private_data;
if (file->f_mode & FMODE_READ) {
struct seq_file *m = file->private_data;
iter = m->private;
} else
iter = file->private_data;
+ if (unlikely(ftrace_disabled))
+ return -ENODEV;
+
+ /* iter->hash is a local copy, so we don't need regex_lock */
+
parser = &iter->parser;
read = trace_get_user(parser, ubuf, cnt, ppos);
parser = &iter->parser;
read = trace_get_user(parser, ubuf, cnt, ppos);
@@
-3276,14
+3307,12
@@
ftrace_regex_write(struct file *file, const char __user *ubuf,
ret = ftrace_process_regex(iter->hash, parser->buffer,
parser->idx, enable);
trace_parser_clear(parser);
ret = ftrace_process_regex(iter->hash, parser->buffer,
parser->idx, enable);
trace_parser_clear(parser);
- if (ret)
- goto out
_unlock
;
+ if (ret
< 0
)
+ goto out;
}
ret = read;
}
ret = read;
-out_unlock:
- mutex_unlock(&ftrace_regex_lock);
-
+ out:
return ret;
}
return ret;
}
@@
-3335,16
+3364,19
@@
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
if (unlikely(ftrace_disabled))
return -ENODEV;
if (unlikely(ftrace_disabled))
return -ENODEV;
+ mutex_lock(&ops->regex_lock);
+
if (enable)
orig_hash = &ops->filter_hash;
else
orig_hash = &ops->notrace_hash;
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
if (enable)
orig_hash = &ops->filter_hash;
else
orig_hash = &ops->notrace_hash;
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
- if (!hash)
- return -ENOMEM;
+ if (!hash) {
+ ret = -ENOMEM;
+ goto out_regex_unlock;
+ }
- mutex_lock(&ftrace_regex_lock);
if (reset)
ftrace_filter_reset(hash);
if (buf && !ftrace_match_records(hash, buf, len)) {
if (reset)
ftrace_filter_reset(hash);
if (buf && !ftrace_match_records(hash, buf, len)) {
@@
-3366,7
+3398,7
@@
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
mutex_unlock(&ftrace_lock);
out_regex_unlock:
mutex_unlock(&ftrace_lock);
out_regex_unlock:
- mutex_unlock(&
ftrace_
regex_lock);
+ mutex_unlock(&
ops->
regex_lock);
free_ftrace_hash(hash);
return ret;
free_ftrace_hash(hash);
return ret;
@@
-3392,6
+3424,7
@@
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
int remove, int reset)
{
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
int remove, int reset)
{
+ ftrace_ops_init(ops);
return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
@@
-3416,6
+3449,7
@@
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset)
{
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset)
{
+ ftrace_ops_init(ops);
return ftrace_set_regex(ops, buf, len, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);
return ftrace_set_regex(ops, buf, len, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);
@@
-3434,6
+3468,7
@@
EXPORT_SYMBOL_GPL(ftrace_set_filter);
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset)
{
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset)
{
+ ftrace_ops_init(ops);
return ftrace_set_regex(ops, buf, len, reset, 0);
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
return ftrace_set_regex(ops, buf, len, reset, 0);
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
@@
-3524,6
+3559,8
@@
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
{
char *func;
{
char *func;
+ ftrace_ops_init(ops);
+
while (buf) {
func = strsep(&buf, ",");
ftrace_set_regex(ops, func, strlen(func), 0, enable);
while (buf) {
func = strsep(&buf, ",");
ftrace_set_regex(ops, func, strlen(func), 0, enable);
@@
-3551,10
+3588,8
@@
int ftrace_regex_release(struct inode *inode, struct file *file)
int filter_hash;
int ret;
int filter_hash;
int ret;
- mutex_lock(&ftrace_regex_lock);
if (file->f_mode & FMODE_READ) {
iter = m->private;
if (file->f_mode & FMODE_READ) {
iter = m->private;
-
seq_release(inode, file);
} else
iter = file->private_data;
seq_release(inode, file);
} else
iter = file->private_data;
@@
-3567,6
+3602,8
@@
int ftrace_regex_release(struct inode *inode, struct file *file)
trace_parser_put(parser);
trace_parser_put(parser);
+ mutex_lock(&iter->ops->regex_lock);
+
if (file->f_mode & FMODE_WRITE) {
filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
if (file->f_mode & FMODE_WRITE) {
filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
@@
-3584,10
+3621,11
@@
int ftrace_regex_release(struct inode *inode, struct file *file)
mutex_unlock(&ftrace_lock);
}
mutex_unlock(&ftrace_lock);
}
+
+ mutex_unlock(&iter->ops->regex_lock);
free_ftrace_hash(iter->hash);
kfree(iter);
free_ftrace_hash(iter->hash);
kfree(iter);
- mutex_unlock(&ftrace_regex_lock);
return 0;
}
return 0;
}
@@
-4126,7
+4164,8
@@
void __init ftrace_init(void)
static struct ftrace_ops global_ops = {
.func = ftrace_stub,
static struct ftrace_ops global_ops = {
.func = ftrace_stub,
- .flags = FTRACE_OPS_FL_RECURSION_SAFE,
+ .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
+ INIT_REGEX_LOCK(global_ops)
};
static int __init ftrace_nodyn_init(void)
};
static int __init ftrace_nodyn_init(void)
@@
-4180,8
+4219,9
@@
ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
}
static struct ftrace_ops control_ops = {
}
static struct ftrace_ops control_ops = {
- .func = ftrace_ops_control_func,
- .flags = FTRACE_OPS_FL_RECURSION_SAFE,
+ .func = ftrace_ops_control_func,
+ .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
+ INIT_REGEX_LOCK(control_ops)
};
static inline void
};
static inline void
@@
-4539,6
+4579,8
@@
int register_ftrace_function(struct ftrace_ops *ops)
{
int ret = -1;
{
int ret = -1;
+ ftrace_ops_init(ops);
+
mutex_lock(&ftrace_lock);
ret = __register_ftrace_function(ops);
mutex_lock(&ftrace_lock);
ret = __register_ftrace_function(ops);