Skip to content

Commit

Permalink
Merge branch 'for-uring-ubufops' of git://git.kernel.org/pub/scm/linu…
Browse files Browse the repository at this point in the history 
…x/kernel/git/kuba/linux into for-6.10/io_uring

Merge net changes required for the upcoming send zerocopy improvements.

* 'for-uring-ubufops' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux:
  net: add callback for setting a ubuf_info to skb
  net: extend ubuf_info callback to ops structure

Signed-off-by: Jens Axboe <[email protected]>
  • Loading branch information
@axboe
axboe committed Apr 23, 2024
2 parents 2f9c951 + 65bada8 commit 3830fff
Show file tree
Hide file tree
Showing 9 changed files with 62 additions and 33 deletions.
2 changes: 1 addition & 1 deletion drivers/net/tap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control,
skb_zcopy_init(skb, msg_control);
} else if (msg_control) {
struct ubuf_info *uarg = msg_control;
uarg->callback(NULL, uarg, false);
uarg->ops->complete(NULL, uarg, false);
}

dev_queue_xmit(skb);
Expand Down
2 changes: 1 addition & 1 deletion drivers/net/tun.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1906,7 +1906,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
skb_zcopy_init(skb, msg_control);
} else if (msg_control) {
struct ubuf_info *uarg = msg_control;
uarg->callback(NULL, uarg, false);
uarg->ops->complete(NULL, uarg, false);
}

skb_reset_network_header(skb);
Expand Down
5 changes: 2 additions & 3 deletions drivers/net/xen-netback/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,9 +390,8 @@ bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);

void xenvif_carrier_on(struct xenvif *vif);

/* Callback from stack when TX packet can be released */
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
bool zerocopy_success);
/* Callbacks from stack when TX packet can be released */
extern const struct ubuf_info_ops xenvif_ubuf_ops;

static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)
{
Expand Down
2 changes: 1 addition & 1 deletion drivers/net/xen-netback/interface.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,7 @@ int xenvif_init_queue(struct xenvif_queue *queue)

for (i = 0; i < MAX_PENDING_REQS; i++) {
queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)
{ { .callback = xenvif_zerocopy_callback },
{ { .ops = &xenvif_ubuf_ops },
{ { .ctx = NULL,
.desc = i } } };
queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;
Expand Down
11 changes: 8 additions & 3 deletions drivers/net/xen-netback/netback.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1156,7 +1156,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s
uarg = skb_shinfo(skb)->destructor_arg;
/* increase inflight counter to offset decrement in callback */
atomic_inc(&queue->inflight_packets);
uarg->callback(NULL, uarg, true);
uarg->ops->complete(NULL, uarg, true);
skb_shinfo(skb)->destructor_arg = NULL;

/* Fill the skb with the new (local) frags. */
Expand Down Expand Up @@ -1278,8 +1278,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue)
return work_done;
}

void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
bool zerocopy_success)
static void xenvif_zerocopy_callback(struct sk_buff *skb,
struct ubuf_info *ubuf_base,
bool zerocopy_success)
{
unsigned long flags;
pending_ring_idx_t index;
Expand Down Expand Up @@ -1312,6 +1313,10 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
xenvif_skb_zerocopy_complete(queue);
}

const struct ubuf_info_ops xenvif_ubuf_ops = {
.complete = xenvif_zerocopy_callback,
};

static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)
{
struct gnttab_unmap_grant_ref *gop;
Expand Down
8 changes: 6 additions & 2 deletions drivers/vhost/net.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net,
}
}

static void vhost_zerocopy_callback(struct sk_buff *skb,
static void vhost_zerocopy_complete(struct sk_buff *skb,
struct ubuf_info *ubuf_base, bool success)
{
struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
Expand Down Expand Up @@ -408,6 +408,10 @@ static void vhost_zerocopy_callback(struct sk_buff *skb,
rcu_read_unlock_bh();
}

static const struct ubuf_info_ops vhost_ubuf_ops = {
.complete = vhost_zerocopy_complete,
};

static inline unsigned long busy_clock(void)
{
return local_clock() >> 10;
Expand Down Expand Up @@ -879,7 +883,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
ubuf->ctx = nvq->ubufs;
ubuf->desc = nvq->upend_idx;
ubuf->ubuf.callback = vhost_zerocopy_callback;
ubuf->ubuf.ops = &vhost_ubuf_ops;
ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
refcount_set(&ubuf->ubuf.refcnt, 1);
msg.msg_control = &ctl;
Expand Down
21 changes: 13 additions & 8 deletions include/linux/skbuff.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,13 @@ enum {
#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)

struct ubuf_info_ops {
void (*complete)(struct sk_buff *, struct ubuf_info *,
bool zerocopy_success);
/* has to be compatible with skb_zcopy_set() */
int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
};

/*
* The callback notifies userspace to release buffers when skb DMA is done in
* lower device, the skb last reference should be 0 when calling this.
Expand All @@ -536,8 +543,7 @@ enum {
* The desc field is used to track userspace buffer index.
*/
struct ubuf_info {
void (*callback)(struct sk_buff *, struct ubuf_info *,
bool zerocopy_success);
const struct ubuf_info_ops *ops;
refcount_t refcnt;
u8 flags;
};
Expand Down Expand Up @@ -1662,14 +1668,13 @@ static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
}
#endif

extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;

struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
struct ubuf_info *uarg);

void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);

void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
bool success);

int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
struct sk_buff *skb, struct iov_iter *from,
size_t length);
Expand Down Expand Up @@ -1757,13 +1762,13 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
static inline void net_zcopy_put(struct ubuf_info *uarg)
{
if (uarg)
uarg->callback(NULL, uarg, true);
uarg->ops->complete(NULL, uarg, true);
}

static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{
if (uarg) {
if (uarg->callback == msg_zerocopy_callback)
if (uarg->ops == &msg_zerocopy_ubuf_ops)
msg_zerocopy_put_abort(uarg, have_uref);
else if (have_uref)
net_zcopy_put(uarg);
Expand All @@ -1777,7 +1782,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)

if (uarg) {
if (!skb_zcopy_is_nouarg(skb))
uarg->callback(skb, uarg, zerocopy_success);
uarg->ops->complete(skb, uarg, zerocopy_success);

skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
}
Expand Down
8 changes: 6 additions & 2 deletions io_uring/notif.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
io_req_task_complete(notif, ts);
}

static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
static void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
bool success)
{
struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
Expand All @@ -43,6 +43,10 @@ static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
__io_req_task_work_add(notif, IOU_F_TWQ_LAZY_WAKE);
}

static const struct ubuf_info_ops io_ubuf_ops = {
.complete = io_tx_ubuf_complete,
};

struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
__must_hold(&ctx->uring_lock)
{
Expand All @@ -62,7 +66,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
nd->zc_report = false;
nd->account_pages = 0;
nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
nd->uarg.callback = io_tx_ubuf_callback;
nd->uarg.ops = &io_ubuf_ops;
refcount_set(&nd->uarg.refcnt, 1);
return notif;
}
36 changes: 24 additions & 12 deletions net/core/skbuff.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1708,7 +1708,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
return NULL;
}

uarg->ubuf.callback = msg_zerocopy_callback;
uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
uarg->len = 1;
uarg->bytelen = size;
Expand All @@ -1734,7 +1734,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
u32 bytelen, next;

/* there might be non MSG_ZEROCOPY users */
if (uarg->callback != msg_zerocopy_callback)
if (uarg->ops != &msg_zerocopy_ubuf_ops)
return NULL;

/* realloc only when socket is locked (TCP, UDP cork),
Expand Down Expand Up @@ -1845,8 +1845,8 @@ static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
sock_put(sk);
}

void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
bool success)
static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
bool success)
{
struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);

Expand All @@ -1855,7 +1855,6 @@ void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
if (refcount_dec_and_test(&uarg->refcnt))
__msg_zerocopy_callback(uarg_zc);
}
EXPORT_SYMBOL_GPL(msg_zerocopy_callback);

void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{
Expand All @@ -1865,22 +1864,34 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
uarg_to_msgzc(uarg)->len--;

if (have_uref)
msg_zerocopy_callback(NULL, uarg, true);
msg_zerocopy_complete(NULL, uarg, true);
}
EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);

const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
.complete = msg_zerocopy_complete,
};
EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);

int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
struct msghdr *msg, int len,
struct ubuf_info *uarg)
{
struct ubuf_info *orig_uarg = skb_zcopy(skb);
int err, orig_len = skb->len;

/* An skb can only point to one uarg. This edge case happens when
* TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
*/
if (orig_uarg && uarg != orig_uarg)
return -EEXIST;
if (uarg->ops->link_skb) {
err = uarg->ops->link_skb(skb, uarg);
if (err)
return err;
} else {
/* An skb can only point to one uarg. This edge case happens
* when TCP appends to an skb, but zerocopy_realloc triggered
* a new alloc.
*/
if (orig_uarg && uarg != orig_uarg)
return -EEXIST;
}

err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
Expand All @@ -1894,7 +1905,8 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
return err;
}

skb_zcopy_set(skb, uarg, NULL);
if (!uarg->ops->link_skb)
skb_zcopy_set(skb, uarg, NULL);
return skb->len - orig_len;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
Expand Down

0 comments on commit 3830fff

Please sign in to comment.