Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

xsk: add indirect call for xsk_destruct_skb 

Since Eric proposed an idea about adding indirect call wrappers for
UDP and managed to see a huge improvement[1], the same situation can
also be applied in xsk scenario.

This patch adds an indirect call for xsk and helps current copy mode
improve the performance by around 1% stably which was observed with
IXGBE at 10Gb/sec loaded. If the throughput grows, the positive effect
will be magnified. I applied this patch on top of batch xmit series[2],
and was able to see <5% improvement from our internal application
which is a little bit unstable though.

Use INDIRECT wrappers to keep xsk_destruct_skb static as it used to
be when the mitigation config is off.

Be aware of the freeing path that can be very hot since the frequency
can reach around 2,000,000 times per second with the xdpsock test.

[1]: https://lore.kernel.org/netdev/20251006193103.2684156-2-edumazet@google.com/
[2]: https://lore.kernel.org/all/20251021131209.41491-1-kerneljasonxing@gmail.com/

Suggested-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Signed-off-by: Jason Xing <kernelxing@tencent.com>
Reviewed-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20251031103328.95468-1-kerneljasonxing@gmail.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Jason Xing 2025-10-31 18:33:28 +08:00 committed by Paolo Abeni
parent b981e100c1
commit 8da7bea7db
3 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/net/xdp_sock.h
View File

@ -125,6 +125,7 @@ struct xsk_tx_metadata_ops {
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
void __xsk_map_flush(struct list_head *flush_list); void __xsk_map_flush(struct list_head *flush_list);
INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *));
/** /**
* xsk_tx_metadata_to_compl - Save enough relevant metadata information * xsk_tx_metadata_to_compl - Save enough relevant metadata information
@ -218,6 +219,12 @@ static inline void __xsk_map_flush(struct list_head *flush_list)
{ {
} }
#ifdef CONFIG_MITIGATION_RETPOLINE
static inline void xsk_destruct_skb(struct sk_buff *skb)
{
}
#endif
static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
struct xsk_tx_metadata_compl *compl) struct xsk_tx_metadata_compl *compl)
{ {

8
net/core/skbuff.c
View File

@ -81,6 +81,7 @@
#include <net/page_pool/helpers.h> #include <net/page_pool/helpers.h>
#include <net/psp/types.h> #include <net/psp/types.h>
#include <net/dropreason.h> #include <net/dropreason.h>
#include <net/xdp_sock.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <trace/events/skb.h> #include <trace/events/skb.h>
@ -1140,12 +1141,13 @@ void skb_release_head_state(struct sk_buff *skb)
if (skb->destructor) { if (skb->destructor) {
DEBUG_NET_WARN_ON_ONCE(in_hardirq()); DEBUG_NET_WARN_ON_ONCE(in_hardirq());
#ifdef CONFIG_INET #ifdef CONFIG_INET
INDIRECT_CALL_3(skb->destructor, INDIRECT_CALL_4(skb->destructor,
tcp_wfree, __sock_wfree, sock_wfree, tcp_wfree, __sock_wfree, sock_wfree,
xsk_destruct_skb,
skb); skb);
#else #else
INDIRECT_CALL_1(skb->destructor, INDIRECT_CALL_2(skb->destructor,
sock_wfree, sock_wfree, xsk_destruct_skb,
skb); skb);
#endif #endif

3
net/xdp/xsk.c
View File

@ -602,7 +602,8 @@ static u32 xsk_get_num_desc(struct sk_buff *skb)
return XSKCB(skb)->num_descs; return XSKCB(skb)->num_descs;
} }
static void xsk_destruct_skb(struct sk_buff *skb) INDIRECT_CALLABLE_SCOPE
void xsk_destruct_skb(struct sk_buff *skb)
{ {
struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta; struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta;