Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Probes for v6.19 

- fprobe: Performance enhancement of the fprobe using rhltable
   . fprobe: use rhltable for fprobe_ip_table. The fprobe IP table has
     been converted to use an rhltable for improved performance when
     dealing with a large number of probed functions.
   . Fix a suspicious RCU usage warning of the above change in the
     fprobe entry handler.
   . Remove an unused local variable of the above change.
   . Fix to initialize fprobe_ip_table in core_initcall().
 
 - fprobe: Performance optimization of fprobe by ftrace
   . fprobe: Use ftrace instead of fgraph for entry only probes. This
     avoids the unneeded overhead of fgraph stack setup.
   . Also update fprobe selftest for entry-only probe.
   . fprobe: Use ftrace only if CONFIG_DYNAMIC_FTRACE_WITH_ARGS or
     WITH_REGS is defined.
 
 - probes: Cleanup probe event subsystems.
   . uprobe/eprobe: Allocate traceprobe_parse_context per probe instead
     of each probe argument parsing. This reduce memory allocation/free
     of temporary working memory.
   . uprobes: Cleanup code using __free().
   . eprobes: Cleanup code using __free().
   . probes: Cleanup code using __free(trace_probe_log_clear) to clear
     error log automatically.
   . probes: Replace strcpy() with memcpy() in __trace_probe_log_err().
 -----BEGIN PGP SIGNATURE-----
 
 iQFPBAABCgA5FiEEh7BulGwFlgAOi5DV2/sHvwUrPxsFAmkvhSsbHG1hc2FtaS5o
 aXJhbWF0c3VAZ21haWwuY29tAAoJENv7B78FKz8bWhEH/23XM5Msjy5vopB+ECZb
 iCj8SkWrQzfiCBILUqxCkZdfJHFomGPHewxvxIOWdb7evtHuy0Ypne/Uw/TMAtAh
 xvDQmu03IV2jO7h7GExsnEh0nX0upYg4IVmN0sCSSWSfgLLTWO9ICClavV9adcva
 ZR+5TdZbK+W59n+ejxA9OMDt1G+nz1Ls9Qhx9ktf7odkJzBkQGPq/heZuPbF3+6k
 Vj2IHTuqWobDDt+ekKOBRWNh9cS61ybxvsr/vmkT6s904ortP6mZa3zEYPRVOUNG
 WJ/KGJwvExTcaG/Dy2g6q8tam1Bidx9/S6klyOGXQXxvaIT1VtBc66HzAUfso6jg
 yIc=
 =w6Kq
 -----END PGP SIGNATURE-----

Merge tag 'probes-v6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull probes updates from Masami Hiramatsu:
 "fprobe performance enhancement using rhltable:
   - use rhltable for fprobe_ip_table. The fprobe IP table has been
     converted to use an rhltable for improved performance when dealing
     with a large number of probed functions
   - Fix a suspicious RCU usage warning of the above change in the
     fprobe entry handler
   - Remove an unused local variable of the above change
   - Fix to initialize fprobe_ip_table in core_initcall()

  Performance optimization of fprobe by ftrace:
   - Use ftrace instead of fgraph for entry only probes. This avoids the
     unneeded overhead of fgraph stack setup
   - Also update fprobe selftest for entry-only probe
   - fprobe: Use ftrace only if CONFIG_DYNAMIC_FTRACE_WITH_ARGS or
     WITH_REGS is defined

  Cleanup probe event subsystems:
   - Allocate traceprobe_parse_context per probe instead of each probe
     argument parsing. This reduce memory allocation/free of temporary
     working memory
   - Cleanup code using __free()
   - Replace strcpy() with memcpy() in __trace_probe_log_err()"

* tag 'probes-v6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace:
  tracing: fprobe: use ftrace if CONFIG_DYNAMIC_FTRACE_WITH_ARGS
  lib/test_fprobe: add testcase for mixed fprobe
  tracing: fprobe: optimization for entry only case
  tracing: fprobe: Fix to init fprobe_ip_table earlier
  tracing: fprobe: Remove unused local variable
  tracing: probes: Replace strcpy() with memcpy() in __trace_probe_log_err()
  tracing: fprobe: fix suspicious rcu usage in fprobe_entry
  tracing: uprobe: eprobes: Allocate traceprobe_parse_context per probe
  tracing: uprobes: Cleanup __trace_uprobe_create() with __free()
  tracing: eprobe: Cleanup eprobe event using __free()
  tracing: probes: Use __free() for trace_probe_log
  tracing: fprobe: use rhltable for fprobe_ip_table
This commit is contained in:
Linus Torvalds 2025-12-05 10:55:47 -08:00
parent 2e8c1c6a50 cd06078a38
commit d1d36025a6
7 changed files with 406 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/linux/fprobe.h
View File

@ -7,6 +7,7 @@
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
#include <linux/refcount.h> #include <linux/refcount.h>
#include <linux/rhashtable.h>
#include <linux/slab.h> #include <linux/slab.h>
struct fprobe; struct fprobe;
@ -26,7 +27,7 @@ typedef void (*fprobe_exit_cb)(struct fprobe *fp, unsigned long entry_ip,
* @fp: The fprobe which owns this. * @fp: The fprobe which owns this.
*/ */
struct fprobe_hlist_node { struct fprobe_hlist_node {
struct hlist_node hlist; struct rhlist_head hlist;
unsigned long addr; unsigned long addr;
struct fprobe *fp; struct fprobe *fp;
}; };

303
kernel/trace/fprobe.c
View File

@ -10,6 +10,7 @@
#include <linux/kprobes.h> #include <linux/kprobes.h>
#include <linux/list.h> #include <linux/list.h>
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/rhashtable.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/sort.h> #include <linux/sort.h>
@ -41,60 +42,68 @@
* - RCU hlist traversal under disabling preempt * - RCU hlist traversal under disabling preempt
*/ */
static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE]; static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
static struct hlist_head fprobe_ip_table[FPROBE_IP_TABLE_SIZE]; static struct rhltable fprobe_ip_table;
static DEFINE_MUTEX(fprobe_mutex); static DEFINE_MUTEX(fprobe_mutex);
static struct fgraph_ops fprobe_graph_ops;
/* static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed)
* Find first fprobe in the hlist. It will be iterated twice in the entry
* probe, once for correcting the total required size, the second time is
* calling back the user handlers.
* Thus the hlist in the fprobe_table must be sorted and new probe needs to
* be added *before* the first fprobe.
*/
static struct fprobe_hlist_node *find_first_fprobe_node(unsigned long ip)
{ {
struct fprobe_hlist_node *node; return hash_ptr(*(unsigned long **)data, 32);
struct hlist_head *head;
head = &fprobe_ip_table[hash_ptr((void *)ip, FPROBE_IP_HASH_BITS)];
hlist_for_each_entry_rcu(node, head, hlist,
lockdep_is_held(&fprobe_mutex)) {
if (node->addr == ip)
return node;
}
return NULL;
} }
NOKPROBE_SYMBOL(find_first_fprobe_node);
static int fprobe_node_cmp(struct rhashtable_compare_arg *arg,
const void *ptr)
{
unsigned long key = *(unsigned long *)arg->key;
const struct fprobe_hlist_node *n = ptr;
return n->addr != key;
}
static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed)
{
const struct fprobe_hlist_node *n = data;
return hash_ptr((void *)n->addr, 32);
}
static const struct rhashtable_params fprobe_rht_params = {
.head_offset = offsetof(struct fprobe_hlist_node, hlist),
.key_offset = offsetof(struct fprobe_hlist_node, addr),
.key_len = sizeof_field(struct fprobe_hlist_node, addr),
.hashfn = fprobe_node_hashfn,
.obj_hashfn = fprobe_node_obj_hashfn,
.obj_cmpfn = fprobe_node_cmp,
.automatic_shrinking = true,
};
/* Node insertion and deletion requires the fprobe_mutex */ /* Node insertion and deletion requires the fprobe_mutex */
static void insert_fprobe_node(struct fprobe_hlist_node *node) static int insert_fprobe_node(struct fprobe_hlist_node *node)
{ {
unsigned long ip = node->addr;
struct fprobe_hlist_node *next;
struct hlist_head *head;
lockdep_assert_held(&fprobe_mutex); lockdep_assert_held(&fprobe_mutex);
next = find_first_fprobe_node(ip); return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params);
if (next) {
hlist_add_before_rcu(&node->hlist, &next->hlist);
return;
}
head = &fprobe_ip_table[hash_ptr((void *)ip, FPROBE_IP_HASH_BITS)];
hlist_add_head_rcu(&node->hlist, head);
} }
/* Return true if there are synonims */ /* Return true if there are synonims */
static bool delete_fprobe_node(struct fprobe_hlist_node *node) static bool delete_fprobe_node(struct fprobe_hlist_node *node)
{ {
lockdep_assert_held(&fprobe_mutex); lockdep_assert_held(&fprobe_mutex);
bool ret;
/* Avoid double deleting */ /* Avoid double deleting */
if (READ_ONCE(node->fp) != NULL) { if (READ_ONCE(node->fp) != NULL) {
WRITE_ONCE(node->fp, NULL); WRITE_ONCE(node->fp, NULL);
hlist_del_rcu(&node->hlist); rhltable_remove(&fprobe_ip_table, &node->hlist,
fprobe_rht_params);
} }
return !!find_first_fprobe_node(node->addr);
rcu_read_lock();
ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr,
fprobe_rht_params);
rcu_read_unlock();
return ret;
} }
/* Check existence of the fprobe */ /* Check existence of the fprobe */
@ -246,12 +255,128 @@ static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent
return ret; return ret;
} }
static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS)
struct ftrace_regs *fregs) /* ftrace_ops callback, this processes fprobes which have only entry_handler. */
static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
struct fprobe_hlist_node *node;
struct rhlist_head *head, *pos;
struct fprobe *fp;
int bit;
bit = ftrace_test_recursion_trylock(ip, parent_ip);
if (bit < 0)
return;
/*
* ftrace_test_recursion_trylock() disables preemption, but
* rhltable_lookup() checks whether rcu_read_lcok is held.
* So we take rcu_read_lock() here.
*/
rcu_read_lock();
head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params);
rhl_for_each_entry_rcu(node, pos, head, hlist) {
if (node->addr != ip)
break;
fp = READ_ONCE(node->fp);
if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler))
continue;
if (fprobe_shared_with_kprobes(fp))
__fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL);
else
__fprobe_handler(ip, parent_ip, fp, fregs, NULL);
}
rcu_read_unlock();
ftrace_test_recursion_unlock(bit);
}
NOKPROBE_SYMBOL(fprobe_ftrace_entry);
static struct ftrace_ops fprobe_ftrace_ops = {
.func = fprobe_ftrace_entry,
.flags = FTRACE_OPS_FL_SAVE_ARGS,
};
static int fprobe_ftrace_active;
static int fprobe_ftrace_add_ips(unsigned long *addrs, int num)
{
int ret;
lockdep_assert_held(&fprobe_mutex);
ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0);
if (ret)
return ret;
if (!fprobe_ftrace_active) {
ret = register_ftrace_function(&fprobe_ftrace_ops);
if (ret) {
ftrace_free_filter(&fprobe_ftrace_ops);
return ret;
}
}
fprobe_ftrace_active++;
return 0;
}
static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num)
{
lockdep_assert_held(&fprobe_mutex);
fprobe_ftrace_active--;
if (!fprobe_ftrace_active)
unregister_ftrace_function(&fprobe_ftrace_ops);
if (num)
ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0);
}
static bool fprobe_is_ftrace(struct fprobe *fp)
{
return !fp->exit_handler;
}
#ifdef CONFIG_MODULES
static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove,
int reset)
{
ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);
ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset);
}
#endif
#else
static int fprobe_ftrace_add_ips(unsigned long *addrs, int num)
{
return -ENOENT;
}
static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num)
{
}
static bool fprobe_is_ftrace(struct fprobe *fp)
{
return false;
}
#ifdef CONFIG_MODULES
static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove,
int reset)
{
ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);
}
#endif
#endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */
/* fgraph_ops callback, this processes fprobes which have exit_handler. */
static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
struct ftrace_regs *fregs)
{ {
struct fprobe_hlist_node *node, *first;
unsigned long *fgraph_data = NULL; unsigned long *fgraph_data = NULL;
unsigned long func = trace->func; unsigned long func = trace->func;
struct fprobe_hlist_node *node;
struct rhlist_head *head, *pos;
unsigned long ret_ip; unsigned long ret_ip;
int reserved_words; int reserved_words;
struct fprobe *fp; struct fprobe *fp;
@ -260,14 +385,12 @@ static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
if (WARN_ON_ONCE(!fregs)) if (WARN_ON_ONCE(!fregs))
return 0; return 0;
first = node = find_first_fprobe_node(func); guard(rcu)();
if (unlikely(!first)) head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params);
return 0;
reserved_words = 0; reserved_words = 0;
hlist_for_each_entry_from_rcu(node, hlist) { rhl_for_each_entry_rcu(node, pos, head, hlist) {
if (node->addr != func) if (node->addr != func)
break; continue;
fp = READ_ONCE(node->fp); fp = READ_ONCE(node->fp);
if (!fp || !fp->exit_handler) if (!fp || !fp->exit_handler)
continue; continue;
@ -278,15 +401,14 @@ static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
reserved_words += reserved_words +=
FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size); FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);
} }
node = first;
if (reserved_words) { if (reserved_words) {
fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long)); fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long));
if (unlikely(!fgraph_data)) { if (unlikely(!fgraph_data)) {
hlist_for_each_entry_from_rcu(node, hlist) { rhl_for_each_entry_rcu(node, pos, head, hlist) {
if (node->addr != func) if (node->addr != func)
break; continue;
fp = READ_ONCE(node->fp); fp = READ_ONCE(node->fp);
if (fp && !fprobe_disabled(fp)) if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp))
fp->nmissed++; fp->nmissed++;
} }
return 0; return 0;
@ -299,14 +421,14 @@ static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
*/ */
ret_ip = ftrace_regs_get_return_address(fregs); ret_ip = ftrace_regs_get_return_address(fregs);
used = 0; used = 0;
hlist_for_each_entry_from_rcu(node, hlist) { rhl_for_each_entry_rcu(node, pos, head, hlist) {
int data_size; int data_size;
void *data; void *data;
if (node->addr != func) if (node->addr != func)
break; continue;
fp = READ_ONCE(node->fp); fp = READ_ONCE(node->fp);
if (!fp || fprobe_disabled(fp)) if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp)))
continue; continue;
data_size = fp->entry_data_size; data_size = fp->entry_data_size;
@ -334,7 +456,7 @@ static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
/* If any exit_handler is set, data must be used. */ /* If any exit_handler is set, data must be used. */
return used != 0; return used != 0;
} }
NOKPROBE_SYMBOL(fprobe_entry); NOKPROBE_SYMBOL(fprobe_fgraph_entry);
static void fprobe_return(struct ftrace_graph_ret *trace, static void fprobe_return(struct ftrace_graph_ret *trace,
struct fgraph_ops *gops, struct fgraph_ops *gops,
@ -373,7 +495,7 @@ static void fprobe_return(struct ftrace_graph_ret *trace,
NOKPROBE_SYMBOL(fprobe_return); NOKPROBE_SYMBOL(fprobe_return);
static struct fgraph_ops fprobe_graph_ops = { static struct fgraph_ops fprobe_graph_ops = {
.entryfunc = fprobe_entry, .entryfunc = fprobe_fgraph_entry,
.retfunc = fprobe_return, .retfunc = fprobe_return,
}; };
static int fprobe_graph_active; static int fprobe_graph_active;
@ -449,25 +571,18 @@ static int fprobe_addr_list_add(struct fprobe_addr_list *alist, unsigned long ad
return 0; return 0;
} }
static void fprobe_remove_node_in_module(struct module *mod, struct hlist_head *head, static void fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node,
struct fprobe_addr_list *alist) struct fprobe_addr_list *alist)
{ {
struct fprobe_hlist_node *node; if (!within_module(node->addr, mod))
int ret = 0; return;
if (delete_fprobe_node(node))
hlist_for_each_entry_rcu(node, head, hlist, return;
lockdep_is_held(&fprobe_mutex)) { /*
if (!within_module(node->addr, mod)) * If failed to update alist, just continue to update hlist.
continue; * Therefore, at list user handler will not hit anymore.
if (delete_fprobe_node(node)) */
continue; fprobe_addr_list_add(alist, node->addr);
/*
* If failed to update alist, just continue to update hlist.
* Therefore, at list user handler will not hit anymore.
*/
if (!ret)
ret = fprobe_addr_list_add(alist, node->addr);
}
} }
/* Handle module unloading to manage fprobe_ip_table. */ /* Handle module unloading to manage fprobe_ip_table. */
@ -475,8 +590,9 @@ static int fprobe_module_callback(struct notifier_block *nb,
unsigned long val, void *data) unsigned long val, void *data)
{ {
struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT}; struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT};
struct fprobe_hlist_node *node;
struct rhashtable_iter iter;
struct module *mod = data; struct module *mod = data;
int i;
if (val != MODULE_STATE_GOING) if (val != MODULE_STATE_GOING)
return NOTIFY_DONE; return NOTIFY_DONE;
@ -487,12 +603,19 @@ static int fprobe_module_callback(struct notifier_block *nb,
return NOTIFY_DONE; return NOTIFY_DONE;
mutex_lock(&fprobe_mutex); mutex_lock(&fprobe_mutex);
for (i = 0; i < FPROBE_IP_TABLE_SIZE; i++) rhltable_walk_enter(&fprobe_ip_table, &iter);
fprobe_remove_node_in_module(mod, &fprobe_ip_table[i], &alist); do {
rhashtable_walk_start(&iter);
while ((node = rhashtable_walk_next(&iter)) && !IS_ERR(node))
fprobe_remove_node_in_module(mod, node, &alist);
rhashtable_walk_stop(&iter);
} while (node == ERR_PTR(-EAGAIN));
rhashtable_walk_exit(&iter);
if (alist.index > 0) if (alist.index > 0)
ftrace_set_filter_ips(&fprobe_graph_ops.ops, fprobe_set_ips(alist.addrs, alist.index, 1, 0);
alist.addrs, alist.index, 1, 0);
mutex_unlock(&fprobe_mutex); mutex_unlock(&fprobe_mutex);
kfree(alist.addrs); kfree(alist.addrs);
@ -725,11 +848,23 @@ int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
mutex_lock(&fprobe_mutex); mutex_lock(&fprobe_mutex);
hlist_array = fp->hlist_array; hlist_array = fp->hlist_array;
ret = fprobe_graph_add_ips(addrs, num); if (fprobe_is_ftrace(fp))
ret = fprobe_ftrace_add_ips(addrs, num);
else
ret = fprobe_graph_add_ips(addrs, num);
if (!ret) { if (!ret) {
add_fprobe_hash(fp); add_fprobe_hash(fp);
for (i = 0; i < hlist_array->size; i++) for (i = 0; i < hlist_array->size; i++) {
insert_fprobe_node(&hlist_array->array[i]); ret = insert_fprobe_node(&hlist_array->array[i]);
if (ret)
break;
}
/* fallback on insert error */
if (ret) {
for (i--; i >= 0; i--)
delete_fprobe_node(&hlist_array->array[i]);
}
} }
mutex_unlock(&fprobe_mutex); mutex_unlock(&fprobe_mutex);
@ -813,7 +948,10 @@ int unregister_fprobe(struct fprobe *fp)
} }
del_fprobe_hash(fp); del_fprobe_hash(fp);
fprobe_graph_remove_ips(addrs, count); if (fprobe_is_ftrace(fp))
fprobe_ftrace_remove_ips(addrs, count);
else
fprobe_graph_remove_ips(addrs, count);
kfree_rcu(hlist_array, rcu); kfree_rcu(hlist_array, rcu);
fp->hlist_array = NULL; fp->hlist_array = NULL;
@ -825,3 +963,10 @@ int unregister_fprobe(struct fprobe *fp)
return ret; return ret;
} }
EXPORT_SYMBOL_GPL(unregister_fprobe); EXPORT_SYMBOL_GPL(unregister_fprobe);
static int __init fprobe_initcall(void)
{
rhltable_init(&fprobe_ip_table, &fprobe_rht_params);
return 0;
}
core_initcall(fprobe_initcall);

108
kernel/trace/trace_eprobe.c
View File

@ -61,6 +61,9 @@ static void trace_event_probe_cleanup(struct trace_eprobe *ep)
kfree(ep); kfree(ep);
} }
DEFINE_FREE(trace_event_probe_cleanup, struct trace_eprobe *,
if (!IS_ERR_OR_NULL(_T)) trace_event_probe_cleanup(_T))
static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev) static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev)
{ {
return container_of(ev, struct trace_eprobe, devent); return container_of(ev, struct trace_eprobe, devent);
@ -197,10 +200,10 @@ static struct trace_eprobe *alloc_event_probe(const char *group,
struct trace_event_call *event, struct trace_event_call *event,
int nargs) int nargs)
{ {
struct trace_eprobe *ep; struct trace_eprobe *ep __free(trace_event_probe_cleanup) = NULL;
const char *event_name; const char *event_name;
const char *sys_name; const char *sys_name;
int ret = -ENOMEM; int ret;
if (!event) if (!event)
return ERR_PTR(-ENODEV); return ERR_PTR(-ENODEV);
@ -211,25 +214,22 @@ static struct trace_eprobe *alloc_event_probe(const char *group,
ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL); ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL);
if (!ep) { if (!ep) {
trace_event_put_ref(event); trace_event_put_ref(event);
goto error; return ERR_PTR(-ENOMEM);
} }
ep->event = event; ep->event = event;
ep->event_name = kstrdup(event_name, GFP_KERNEL); ep->event_name = kstrdup(event_name, GFP_KERNEL);
if (!ep->event_name) if (!ep->event_name)
goto error; return ERR_PTR(-ENOMEM);
ep->event_system = kstrdup(sys_name, GFP_KERNEL); ep->event_system = kstrdup(sys_name, GFP_KERNEL);
if (!ep->event_system) if (!ep->event_system)
goto error; return ERR_PTR(-ENOMEM);
ret = trace_probe_init(&ep->tp, this_event, group, false, nargs); ret = trace_probe_init(&ep->tp, this_event, group, false, nargs);
if (ret < 0) if (ret < 0)
goto error; return ERR_PTR(ret);
dyn_event_init(&ep->devent, &eprobe_dyn_event_ops); dyn_event_init(&ep->devent, &eprobe_dyn_event_ops);
return ep; return_ptr(ep);
error:
trace_event_probe_cleanup(ep);
return ERR_PTR(ret);
} }
static int eprobe_event_define_fields(struct trace_event_call *event_call) static int eprobe_event_define_fields(struct trace_event_call *event_call)
@ -790,25 +790,6 @@ find_and_get_event(const char *system, const char *event_name)
return NULL; return NULL;
} }
static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i)
{
struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
int ret;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
ctx->event = ep->event;
ctx->flags = TPARG_FL_KERNEL | TPARG_FL_TEVENT;
ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], ctx);
/* Handle symbols "@" */
if (!ret)
ret = traceprobe_update_arg(&ep->tp.args[i]);
return ret;
}
static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[]) static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[])
{ {
struct event_filter *dummy = NULL; struct event_filter *dummy = NULL;
@ -845,13 +826,10 @@ static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const ch
ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str, ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
true, &dummy); true, &dummy);
free_event_filter(dummy); free_event_filter(dummy);
if (ret) if (ret) {
goto error; kfree(ep->filter_str);
ep->filter_str = NULL;
return 0; }
error:
kfree(ep->filter_str);
ep->filter_str = NULL;
return ret; return ret;
} }
@ -863,31 +841,33 @@ static int __trace_eprobe_create(int argc, const char *argv[])
* Fetch args (no space): * Fetch args (no space):
* <name>=$<field>[:TYPE] * <name>=$<field>[:TYPE]
*/ */
struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
struct trace_eprobe *ep __free(trace_event_probe_cleanup) = NULL;
const char *trlog __free(trace_probe_log_clear) = NULL;
const char *event = NULL, *group = EPROBE_EVENT_SYSTEM; const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
const char *sys_event = NULL, *sys_name = NULL; const char *sys_event = NULL, *sys_name = NULL;
struct trace_event_call *event_call; struct trace_event_call *event_call;
char *buf1 __free(kfree) = NULL; char *buf1 __free(kfree) = NULL;
char *buf2 __free(kfree) = NULL; char *buf2 __free(kfree) = NULL;
char *gbuf __free(kfree) = NULL; char *gbuf __free(kfree) = NULL;
struct trace_eprobe *ep = NULL;
int ret = 0, filter_idx = 0; int ret = 0, filter_idx = 0;
int i, filter_cnt; int i, filter_cnt;
if (argc < 2 || argv[0][0] != 'e') if (argc < 2 || argv[0][0] != 'e')
return -ECANCELED; return -ECANCELED;
trace_probe_log_init("event_probe", argc, argv); trlog = trace_probe_log_init("event_probe", argc, argv);
event = strchr(&argv[0][1], ':'); event = strchr(&argv[0][1], ':');
if (event) { if (event) {
gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
if (!gbuf) if (!gbuf)
goto mem_error; return -ENOMEM;
event++; event++;
ret = traceprobe_parse_event_name(&event, &group, gbuf, ret = traceprobe_parse_event_name(&event, &group, gbuf,
event - argv[0]); event - argv[0]);
if (ret) if (ret)
goto parse_error; return -EINVAL;
} }
trace_probe_log_set_index(1); trace_probe_log_set_index(1);
@ -895,18 +875,18 @@ static int __trace_eprobe_create(int argc, const char *argv[])
buf2 = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); buf2 = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
if (!buf2) if (!buf2)
goto mem_error; return -ENOMEM;
ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0); ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0);
if (ret || !sys_event || !sys_name) { if (ret || !sys_event || !sys_name) {
trace_probe_log_err(0, NO_EVENT_INFO); trace_probe_log_err(0, NO_EVENT_INFO);
goto parse_error; return -EINVAL;
} }
if (!event) { if (!event) {
buf1 = kstrdup(sys_event, GFP_KERNEL); buf1 = kstrdup(sys_event, GFP_KERNEL);
if (!buf1) if (!buf1)
goto mem_error; return -ENOMEM;
event = buf1; event = buf1;
} }
@ -922,8 +902,7 @@ static int __trace_eprobe_create(int argc, const char *argv[])
if (argc - 2 > MAX_TRACE_ARGS) { if (argc - 2 > MAX_TRACE_ARGS) {
trace_probe_log_set_index(2); trace_probe_log_set_index(2);
trace_probe_log_err(0, TOO_MANY_ARGS); trace_probe_log_err(0, TOO_MANY_ARGS);
ret = -E2BIG; return -E2BIG;
goto error;
} }
scoped_guard(mutex, &event_mutex) { scoped_guard(mutex, &event_mutex) {
@ -937,29 +916,39 @@ static int __trace_eprobe_create(int argc, const char *argv[])
trace_probe_log_err(0, BAD_ATTACH_EVENT); trace_probe_log_err(0, BAD_ATTACH_EVENT);
/* This must return -ENOMEM or missing event, else there is a bug */ /* This must return -ENOMEM or missing event, else there is a bug */
WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV); WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV);
ep = NULL; return ret;
goto error;
} }
if (filter_idx) { if (filter_idx) {
trace_probe_log_set_index(filter_idx); trace_probe_log_set_index(filter_idx);
ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx); ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
if (ret) if (ret)
goto parse_error; return -EINVAL;
} else } else
ep->filter_str = NULL; ep->filter_str = NULL;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
ctx->event = ep->event;
ctx->flags = TPARG_FL_KERNEL | TPARG_FL_TEVENT;
argc -= 2; argv += 2; argc -= 2; argv += 2;
/* parse arguments */ /* parse arguments */
for (i = 0; i < argc; i++) { for (i = 0; i < argc; i++) {
trace_probe_log_set_index(i + 2); trace_probe_log_set_index(i + 2);
ret = trace_eprobe_tp_update_arg(ep, argv, i);
ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], ctx);
/* Handle symbols "@" */
if (!ret)
ret = traceprobe_update_arg(&ep->tp.args[i]);
if (ret) if (ret)
goto error; return ret;
} }
ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT); ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT);
if (ret < 0) if (ret < 0)
goto error; return ret;
init_trace_eprobe_call(ep); init_trace_eprobe_call(ep);
scoped_guard(mutex, &event_mutex) { scoped_guard(mutex, &event_mutex) {
ret = trace_probe_register_event_call(&ep->tp); ret = trace_probe_register_event_call(&ep->tp);
@ -968,25 +957,16 @@ static int __trace_eprobe_create(int argc, const char *argv[])
trace_probe_log_set_index(0); trace_probe_log_set_index(0);
trace_probe_log_err(0, EVENT_EXIST); trace_probe_log_err(0, EVENT_EXIST);
} }
goto error; return ret;
} }
ret = dyn_event_add(&ep->devent, &ep->tp.event->call); ret = dyn_event_add(&ep->devent, &ep->tp.event->call);
if (ret < 0) { if (ret < 0) {
trace_probe_unregister_event_call(&ep->tp); trace_probe_unregister_event_call(&ep->tp);
goto error; return ret;
} }
/* To avoid freeing registered eprobe event, clear ep. */
ep = NULL;
} }
trace_probe_log_clear();
return ret;
mem_error:
ret = -ENOMEM;
goto error;
parse_error:
ret = -EINVAL;
error:
trace_probe_log_clear();
trace_event_probe_cleanup(ep);
return ret; return ret;
} }

5
kernel/trace/trace_probe.c
View File

@ -156,7 +156,7 @@ static const struct fetch_type *find_fetch_type(const char *type, unsigned long
static struct trace_probe_log trace_probe_log; static struct trace_probe_log trace_probe_log;
extern struct mutex dyn_event_ops_mutex; extern struct mutex dyn_event_ops_mutex;
void trace_probe_log_init(const char *subsystem, int argc, const char **argv) const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv)
{ {
lockdep_assert_held(&dyn_event_ops_mutex); lockdep_assert_held(&dyn_event_ops_mutex);
@ -164,6 +164,7 @@ void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
trace_probe_log.argc = argc; trace_probe_log.argc = argc;
trace_probe_log.argv = argv; trace_probe_log.argv = argv;
trace_probe_log.index = 0; trace_probe_log.index = 0;
return subsystem;
} }
void trace_probe_log_clear(void) void trace_probe_log_clear(void)
@ -214,7 +215,7 @@ void __trace_probe_log_err(int offset, int err_type)
p = command; p = command;
for (i = 0; i < trace_probe_log.argc; i++) { for (i = 0; i < trace_probe_log.argc; i++) {
len = strlen(trace_probe_log.argv[i]); len = strlen(trace_probe_log.argv[i]);
strcpy(p, trace_probe_log.argv[i]); memcpy(p, trace_probe_log.argv[i], len);
p[len] = ' '; p[len] = ' ';
p += len + 1; p += len + 1;
} }

4
kernel/trace/trace_probe.h
View File

@ -578,11 +578,13 @@ struct trace_probe_log {
int index; int index;
}; };
void trace_probe_log_init(const char *subsystem, int argc, const char **argv); const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv);
void trace_probe_log_set_index(int index); void trace_probe_log_set_index(int index);
void trace_probe_log_clear(void); void trace_probe_log_clear(void);
void __trace_probe_log_err(int offset, int err); void __trace_probe_log_err(int offset, int err);
DEFINE_FREE(trace_probe_log_clear, const char *, if (_T) trace_probe_log_clear())
#define trace_probe_log_err(offs, err) \ #define trace_probe_log_err(offs, err) \
__trace_probe_log_err(offs, TP_ERR_##err) __trace_probe_log_err(offs, TP_ERR_##err)

82
kernel/trace/trace_uprobe.c
View File

@ -533,21 +533,26 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
return ret; return ret;
} }
DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, if (_T) free_trace_uprobe(_T))
/* /*
* Argument syntax: * Argument syntax:
* - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS] * - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
*/ */
static int __trace_uprobe_create(int argc, const char **argv) static int __trace_uprobe_create(int argc, const char **argv)
{ {
struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
struct trace_uprobe *tu __free(free_trace_uprobe) = NULL;
const char *trlog __free(trace_probe_log_clear) = NULL;
const char *event = NULL, *group = UPROBE_EVENT_SYSTEM; const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
char *arg, *filename, *rctr, *rctr_end, *tmp; struct path path __free(path_put) = {};
unsigned long offset, ref_ctr_offset; unsigned long offset, ref_ctr_offset;
char *filename __free(kfree) = NULL;
char *arg, *rctr, *rctr_end, *tmp;
char *gbuf __free(kfree) = NULL; char *gbuf __free(kfree) = NULL;
char *buf __free(kfree) = NULL; char *buf __free(kfree) = NULL;
enum probe_print_type ptype; enum probe_print_type ptype;
struct trace_uprobe *tu;
bool is_return = false; bool is_return = false;
struct path path;
int i, ret; int i, ret;
ref_ctr_offset = 0; ref_ctr_offset = 0;
@ -565,7 +570,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
if (argc < 2) if (argc < 2)
return -ECANCELED; return -ECANCELED;
trace_probe_log_init("trace_uprobe", argc, argv); trlog = trace_probe_log_init("trace_uprobe", argc, argv);
if (argc - 2 > MAX_TRACE_ARGS) { if (argc - 2 > MAX_TRACE_ARGS) {
trace_probe_log_set_index(2); trace_probe_log_set_index(2);
@ -585,10 +590,8 @@ static int __trace_uprobe_create(int argc, const char **argv)
/* Find the last occurrence, in case the path contains ':' too. */ /* Find the last occurrence, in case the path contains ':' too. */
arg = strrchr(filename, ':'); arg = strrchr(filename, ':');
if (!arg || !isdigit(arg[1])) { if (!arg || !isdigit(arg[1]))
kfree(filename);
return -ECANCELED; return -ECANCELED;
}
trace_probe_log_set_index(1); /* filename is the 2nd argument */ trace_probe_log_set_index(1); /* filename is the 2nd argument */
@ -596,14 +599,11 @@ static int __trace_uprobe_create(int argc, const char **argv)
ret = kern_path(filename, LOOKUP_FOLLOW, &path); ret = kern_path(filename, LOOKUP_FOLLOW, &path);
if (ret) { if (ret) {
trace_probe_log_err(0, FILE_NOT_FOUND); trace_probe_log_err(0, FILE_NOT_FOUND);
kfree(filename);
trace_probe_log_clear();
return ret; return ret;
} }
if (!d_is_reg(path.dentry)) { if (!d_is_reg(path.dentry)) {
trace_probe_log_err(0, NO_REGULAR_FILE); trace_probe_log_err(0, NO_REGULAR_FILE);
ret = -EINVAL; return -EINVAL;
goto fail_address_parse;
} }
/* Parse reference counter offset if specified. */ /* Parse reference counter offset if specified. */
@ -611,16 +611,14 @@ static int __trace_uprobe_create(int argc, const char **argv)
if (rctr) { if (rctr) {
rctr_end = strchr(rctr, ')'); rctr_end = strchr(rctr, ')');
if (!rctr_end) { if (!rctr_end) {
ret = -EINVAL;
rctr_end = rctr + strlen(rctr); rctr_end = rctr + strlen(rctr);
trace_probe_log_err(rctr_end - filename, trace_probe_log_err(rctr_end - filename,
REFCNT_OPEN_BRACE); REFCNT_OPEN_BRACE);
goto fail_address_parse; return -EINVAL;
} else if (rctr_end[1] != '\0') { } else if (rctr_end[1] != '\0') {
ret = -EINVAL;
trace_probe_log_err(rctr_end + 1 - filename, trace_probe_log_err(rctr_end + 1 - filename,
BAD_REFCNT_SUFFIX); BAD_REFCNT_SUFFIX);
goto fail_address_parse; return -EINVAL;
} }
*rctr++ = '\0'; *rctr++ = '\0';
@ -628,7 +626,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
ret = kstrtoul(rctr, 0, &ref_ctr_offset); ret = kstrtoul(rctr, 0, &ref_ctr_offset);
if (ret) { if (ret) {
trace_probe_log_err(rctr - filename, BAD_REFCNT); trace_probe_log_err(rctr - filename, BAD_REFCNT);
goto fail_address_parse; return ret;
} }
} }
@ -640,8 +638,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
is_return = true; is_return = true;
} else { } else {
trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX); trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
ret = -EINVAL; return -EINVAL;
goto fail_address_parse;
} }
} }
@ -649,7 +646,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
ret = kstrtoul(arg, 0, &offset); ret = kstrtoul(arg, 0, &offset);
if (ret) { if (ret) {
trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS); trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
goto fail_address_parse; return ret;
} }
/* setup a probe */ /* setup a probe */
@ -657,12 +654,12 @@ static int __trace_uprobe_create(int argc, const char **argv)
if (event) { if (event) {
gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
if (!gbuf) if (!gbuf)
goto fail_mem; return -ENOMEM;
ret = traceprobe_parse_event_name(&event, &group, gbuf, ret = traceprobe_parse_event_name(&event, &group, gbuf,
event - argv[0]); event - argv[0]);
if (ret) if (ret)
goto fail_address_parse; return ret;
} }
if (!event) { if (!event) {
@ -671,7 +668,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
tail = kstrdup(kbasename(filename), GFP_KERNEL); tail = kstrdup(kbasename(filename), GFP_KERNEL);
if (!tail) if (!tail)
goto fail_mem; return -ENOMEM;
ptr = strpbrk(tail, ".-_"); ptr = strpbrk(tail, ".-_");
if (ptr) if (ptr)
@ -679,7 +676,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
if (!buf) if (!buf)
goto fail_mem; return -ENOMEM;
snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset); snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
event = buf; event = buf;
kfree(tail); kfree(tail);
@ -693,51 +690,36 @@ static int __trace_uprobe_create(int argc, const char **argv)
ret = PTR_ERR(tu); ret = PTR_ERR(tu);
/* This must return -ENOMEM otherwise there is a bug */ /* This must return -ENOMEM otherwise there is a bug */
WARN_ON_ONCE(ret != -ENOMEM); WARN_ON_ONCE(ret != -ENOMEM);
goto fail_address_parse; return ret;
} }
tu->offset = offset; tu->offset = offset;
tu->ref_ctr_offset = ref_ctr_offset; tu->ref_ctr_offset = ref_ctr_offset;
tu->path = path; tu->path = path;
tu->filename = filename; /* Clear @path so that it will not freed by path_put() */
memset(&path, 0, sizeof(path));
tu->filename = no_free_ptr(filename);
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER;
/* parse arguments */ /* parse arguments */
for (i = 0; i < argc; i++) { for (i = 0; i < argc; i++) {
struct traceprobe_parse_context *ctx __free(traceprobe_parse_context)
= kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx) {
ret = -ENOMEM;
goto error;
}
ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER;
trace_probe_log_set_index(i + 2); trace_probe_log_set_index(i + 2);
ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx); ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx);
if (ret) if (ret)
goto error; return ret;
} }
ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
ret = traceprobe_set_print_fmt(&tu->tp, ptype); ret = traceprobe_set_print_fmt(&tu->tp, ptype);
if (ret < 0) if (ret < 0)
goto error; return ret;
ret = register_trace_uprobe(tu); ret = register_trace_uprobe(tu);
if (!ret) if (!ret)
goto out; tu = NULL;
error:
free_trace_uprobe(tu);
out:
trace_probe_log_clear();
return ret;
fail_mem:
ret = -ENOMEM;
fail_address_parse:
trace_probe_log_clear();
path_put(&path);
kfree(filename);
return ret; return ret;
} }

99
lib/tests/test_fprobe.c
View File

@ -12,7 +12,8 @@
static struct kunit *current_test; static struct kunit *current_test;
static u32 rand1, entry_val, exit_val; static u32 rand1, entry_only_val, entry_val, exit_val;
static u32 entry_only_count, entry_count, exit_count;
/* Use indirect calls to avoid inlining the target functions */ /* Use indirect calls to avoid inlining the target functions */
static u32 (*target)(u32 value); static u32 (*target)(u32 value);
@ -190,6 +191,101 @@ static void test_fprobe_skip(struct kunit *test)
KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp));
} }
/* Handler for fprobe entry only case */
static notrace int entry_only_handler(struct fprobe *fp, unsigned long ip,
unsigned long ret_ip,
struct ftrace_regs *fregs, void *data)
{
KUNIT_EXPECT_FALSE(current_test, preemptible());
KUNIT_EXPECT_EQ(current_test, ip, target_ip);
entry_only_count++;
entry_only_val = (rand1 / div_factor);
return 0;
}
static notrace int fprobe_entry_multi_handler(struct fprobe *fp, unsigned long ip,
unsigned long ret_ip,
struct ftrace_regs *fregs,
void *data)
{
KUNIT_EXPECT_FALSE(current_test, preemptible());
KUNIT_EXPECT_EQ(current_test, ip, target_ip);
entry_count++;
entry_val = (rand1 / div_factor);
return 0;
}
static notrace void fprobe_exit_multi_handler(struct fprobe *fp, unsigned long ip,
unsigned long ret_ip,
struct ftrace_regs *fregs,
void *data)
{
unsigned long ret = ftrace_regs_get_return_value(fregs);
KUNIT_EXPECT_FALSE(current_test, preemptible());
KUNIT_EXPECT_EQ(current_test, ip, target_ip);
KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor));
exit_count++;
exit_val = ret;
}
static void check_fprobe_multi(struct kunit *test)
{
entry_only_count = entry_count = exit_count = 0;
entry_only_val = entry_val = exit_val = 0;
target(rand1);
/* Verify all handlers were called */
KUNIT_EXPECT_EQ(test, 1, entry_only_count);
KUNIT_EXPECT_EQ(test, 1, entry_count);
KUNIT_EXPECT_EQ(test, 1, exit_count);
/* Verify values are correct */
KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_only_val);
KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_val);
KUNIT_EXPECT_EQ(test, (rand1 / div_factor), exit_val);
}
/* Test multiple fprobes hooking the same target function */
static void test_fprobe_multi(struct kunit *test)
{
struct fprobe fp1 = {
.entry_handler = fprobe_entry_multi_handler,
.exit_handler = fprobe_exit_multi_handler,
};
struct fprobe fp2 = {
.entry_handler = entry_only_handler,
};
current_test = test;
/* Test Case 1: Register in order 1 -> 2 */
KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL));
KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL));
check_fprobe_multi(test);
/* Unregister all */
KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1));
KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2));
/* Test Case 2: Register in order 2 -> 1 */
KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL));
KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL));
check_fprobe_multi(test);
/* Unregister all */
KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1));
KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2));
}
static unsigned long get_ftrace_location(void *func) static unsigned long get_ftrace_location(void *func)
{ {
unsigned long size, addr = (unsigned long)func; unsigned long size, addr = (unsigned long)func;
@ -217,6 +313,7 @@ static struct kunit_case fprobe_testcases[] = {
KUNIT_CASE(test_fprobe_syms), KUNIT_CASE(test_fprobe_syms),
KUNIT_CASE(test_fprobe_data), KUNIT_CASE(test_fprobe_data),
KUNIT_CASE(test_fprobe_skip), KUNIT_CASE(test_fprobe_skip),
KUNIT_CASE(test_fprobe_multi),
{} {}
}; };