2018-05-02 19:01:23 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
/* XDP sockets
|
|
|
|
|
*
|
|
|
|
|
* AF_XDP sockets allows a channel between XDP programs and userspace
|
|
|
|
|
* applications.
|
|
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
|
*
|
|
|
|
|
* Author(s): Björn Töpel <bjorn.topel@intel.com>
|
|
|
|
|
* Magnus Karlsson <magnus.karlsson@intel.com>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
|
|
|
|
|
|
|
|
|
|
#include <linux/if_xdp.h>
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
|
#include <linux/sched/mm.h>
|
|
|
|
|
#include <linux/sched/signal.h>
|
|
|
|
|
#include <linux/sched/task.h>
|
|
|
|
|
#include <linux/socket.h>
|
|
|
|
|
#include <linux/file.h>
|
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
|
#include <linux/net.h>
|
|
|
|
|
#include <linux/netdevice.h>
|
2018-06-04 20:05:57 +08:00
|
|
|
#include <linux/rculist.h>
|
2018-05-02 19:01:23 +08:00
|
|
|
#include <net/xdp_sock.h>
|
2018-05-02 19:01:25 +08:00
|
|
|
#include <net/xdp.h>
|
2018-05-02 19:01:23 +08:00
|
|
|
|
2018-05-02 19:01:24 +08:00
|
|
|
#include "xsk_queue.h"
|
2018-05-02 19:01:23 +08:00
|
|
|
#include "xdp_umem.h"
|
|
|
|
|
|
2018-05-02 19:01:34 +08:00
|
|
|
#define TX_BATCH_SIZE 16
|
|
|
|
|
|
2018-05-02 19:01:23 +08:00
|
|
|
static struct xdp_sock *xdp_sk(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
return (struct xdp_sock *)sk;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:28 +08:00
|
|
|
bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
|
|
|
|
|
{
|
2018-06-04 20:05:55 +08:00
|
|
|
return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) &&
|
|
|
|
|
READ_ONCE(xs->umem->fq);
|
2018-05-02 19:01:28 +08:00
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
|
|
|
|
|
{
|
|
|
|
|
return xskq_peek_addr(umem->fq, addr);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_peek_addr);
|
|
|
|
|
|
|
|
|
|
void xsk_umem_discard_addr(struct xdp_umem *umem)
|
|
|
|
|
{
|
|
|
|
|
xskq_discard_addr(umem->fq);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_discard_addr);
|
|
|
|
|
|
|
|
|
|
static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
|
|
|
|
void *buffer;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
u64 addr;
|
2018-06-04 19:57:11 +08:00
|
|
|
int err;
|
2018-05-02 19:01:27 +08:00
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
if (!xskq_peek_addr(xs->umem->fq, &addr) ||
|
|
|
|
|
len > xs->umem->chunk_size_nohr) {
|
2018-06-04 19:57:12 +08:00
|
|
|
xs->rx_dropped++;
|
2018-05-02 19:01:27 +08:00
|
|
|
return -ENOSPC;
|
2018-06-04 19:57:12 +08:00
|
|
|
}
|
2018-05-02 19:01:27 +08:00
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
addr += xs->umem->headroom;
|
|
|
|
|
|
|
|
|
|
buffer = xdp_umem_get_data(xs->umem, addr);
|
2018-05-02 19:01:27 +08:00
|
|
|
memcpy(buffer, xdp->data, len);
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
err = xskq_produce_batch_desc(xs->rx, addr, len);
|
2018-06-04 20:05:55 +08:00
|
|
|
if (!err) {
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
xskq_discard_addr(xs->umem->fq);
|
2018-06-04 20:05:55 +08:00
|
|
|
xdp_return_buff(xdp);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 19:01:27 +08:00
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
xs->rx_dropped++;
|
2018-05-02 19:01:27 +08:00
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
2018-06-04 20:05:55 +08:00
|
|
|
int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);
|
2018-05-02 19:01:27 +08:00
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
if (err) {
|
2018-05-02 19:01:27 +08:00
|
|
|
xdp_return_buff(xdp);
|
2018-06-04 20:05:55 +08:00
|
|
|
xs->rx_dropped++;
|
|
|
|
|
}
|
2018-05-02 19:01:27 +08:00
|
|
|
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
|
|
|
|
|
{
|
|
|
|
|
u32 len;
|
|
|
|
|
|
|
|
|
|
if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
len = xdp->data_end - xdp->data;
|
|
|
|
|
|
|
|
|
|
return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
|
|
|
|
|
__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:27 +08:00
|
|
|
void xsk_flush(struct xdp_sock *xs)
|
|
|
|
|
{
|
|
|
|
|
xskq_produce_flush_desc(xs->rx);
|
|
|
|
|
xs->sk.sk_data_ready(&xs->sk);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
|
|
|
|
|
{
|
2018-06-04 20:05:55 +08:00
|
|
|
u32 len = xdp->data_end - xdp->data;
|
|
|
|
|
void *buffer;
|
|
|
|
|
u64 addr;
|
2018-05-02 19:01:27 +08:00
|
|
|
int err;
|
|
|
|
|
|
2018-06-12 18:02:56 +08:00
|
|
|
if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
if (!xskq_peek_addr(xs->umem->fq, &addr) ||
|
|
|
|
|
len > xs->umem->chunk_size_nohr) {
|
|
|
|
|
xs->rx_dropped++;
|
|
|
|
|
return -ENOSPC;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
addr += xs->umem->headroom;
|
|
|
|
|
|
|
|
|
|
buffer = xdp_umem_get_data(xs->umem, addr);
|
|
|
|
|
memcpy(buffer, xdp->data, len);
|
|
|
|
|
err = xskq_produce_batch_desc(xs->rx, addr, len);
|
|
|
|
|
if (!err) {
|
|
|
|
|
xskq_discard_addr(xs->umem->fq);
|
2018-05-02 19:01:27 +08:00
|
|
|
xsk_flush(xs);
|
2018-06-04 20:05:55 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 19:01:27 +08:00
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
xs->rx_dropped++;
|
2018-05-02 19:01:27 +08:00
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:57 +08:00
|
|
|
void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
|
|
|
|
|
{
|
|
|
|
|
xskq_produce_flush_addr_n(umem->cq, nb_entries);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_complete_tx);
|
|
|
|
|
|
|
|
|
|
void xsk_umem_consume_tx_done(struct xdp_umem *umem)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
|
|
|
|
|
xs->sk.sk_write_space(&xs->sk);
|
|
|
|
|
}
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_consume_tx_done);
|
|
|
|
|
|
|
|
|
|
bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_desc desc;
|
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
|
|
|
|
|
if (!xskq_peek_desc(xs->tx, &desc))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
if (xskq_produce_addr_lazy(umem->cq, desc.addr))
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
*dma = xdp_umem_get_dma(umem, desc.addr);
|
|
|
|
|
*len = desc.len;
|
|
|
|
|
|
|
|
|
|
xskq_discard_desc(xs->tx);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(xsk_umem_consume_tx);
|
|
|
|
|
|
|
|
|
|
static int xsk_zc_xmit(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
struct net_device *dev = xs->dev;
|
|
|
|
|
|
|
|
|
|
return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:34 +08:00
|
|
|
static void xsk_destruct_skb(struct sk_buff *skb)
|
|
|
|
|
{
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
|
2018-05-02 19:01:34 +08:00
|
|
|
struct xdp_sock *xs = xdp_sk(skb->sk);
|
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr));
|
2018-05-02 19:01:34 +08:00
|
|
|
|
|
|
|
|
sock_wfree(skb);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_generic_xmit(struct sock *sk, struct msghdr *m,
|
|
|
|
|
size_t total_len)
|
|
|
|
|
{
|
|
|
|
|
u32 max_batch = TX_BATCH_SIZE;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
bool sent_frame = false;
|
|
|
|
|
struct xdp_desc desc;
|
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
|
|
if (unlikely(!xs->tx))
|
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
|
|
|
|
|
|
while (xskq_peek_desc(xs->tx, &desc)) {
|
|
|
|
|
char *buffer;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
u64 addr;
|
|
|
|
|
u32 len;
|
2018-05-02 19:01:34 +08:00
|
|
|
|
|
|
|
|
if (max_batch-- == 0) {
|
|
|
|
|
err = -EAGAIN;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
if (xskq_reserve_addr(xs->umem->cq)) {
|
2018-05-02 19:01:34 +08:00
|
|
|
err = -EAGAIN;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
len = desc.len;
|
|
|
|
|
if (unlikely(len > xs->dev->mtu)) {
|
|
|
|
|
err = -EMSGSIZE;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-22 15:34:58 +08:00
|
|
|
if (xs->queue_id >= xs->dev->real_num_tx_queues) {
|
|
|
|
|
err = -ENXIO;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:57 +08:00
|
|
|
skb = sock_alloc_send_skb(sk, len, 1, &err);
|
2018-05-02 19:01:34 +08:00
|
|
|
if (unlikely(!skb)) {
|
|
|
|
|
err = -EAGAIN;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
skb_put(skb, len);
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
addr = desc.addr;
|
|
|
|
|
buffer = xdp_umem_get_data(xs->umem, addr);
|
2018-05-02 19:01:34 +08:00
|
|
|
err = skb_store_bits(skb, 0, buffer, len);
|
|
|
|
|
if (unlikely(err)) {
|
|
|
|
|
kfree_skb(skb);
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
skb->dev = xs->dev;
|
|
|
|
|
skb->priority = sk->sk_priority;
|
|
|
|
|
skb->mark = sk->sk_mark;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
skb_shinfo(skb)->destructor_arg = (void *)(long)addr;
|
2018-05-02 19:01:34 +08:00
|
|
|
skb->destructor = xsk_destruct_skb;
|
|
|
|
|
|
|
|
|
|
err = dev_direct_xmit(skb, xs->queue_id);
|
2018-06-29 15:48:18 +08:00
|
|
|
xskq_discard_desc(xs->tx);
|
2018-05-02 19:01:34 +08:00
|
|
|
/* Ignore NET_XMIT_CN as packet might have been sent */
|
|
|
|
|
if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
|
2018-06-29 15:48:18 +08:00
|
|
|
/* SKB completed but not sent */
|
|
|
|
|
err = -EBUSY;
|
2018-05-02 19:01:34 +08:00
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sent_frame = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
if (sent_frame)
|
|
|
|
|
sk->sk_write_space(sk);
|
|
|
|
|
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
|
|
|
|
|
{
|
2018-06-04 20:05:57 +08:00
|
|
|
bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
|
2018-05-02 19:01:34 +08:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
|
|
if (unlikely(!xs->dev))
|
|
|
|
|
return -ENXIO;
|
|
|
|
|
if (unlikely(!(xs->dev->flags & IFF_UP)))
|
|
|
|
|
return -ENETDOWN;
|
2018-06-04 20:05:57 +08:00
|
|
|
if (need_wait)
|
|
|
|
|
return -EOPNOTSUPP;
|
2018-05-02 19:01:34 +08:00
|
|
|
|
2018-06-04 20:05:57 +08:00
|
|
|
return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len);
|
2018-05-02 19:01:34 +08:00
|
|
|
}
|
|
|
|
|
|
2018-06-29 00:43:44 +08:00
|
|
|
static unsigned int xsk_poll(struct file *file, struct socket *sock,
|
|
|
|
|
struct poll_table_struct *wait)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
2018-06-29 00:43:44 +08:00
|
|
|
unsigned int mask = datagram_poll(file, sock, wait);
|
2018-05-02 19:01:27 +08:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
|
|
if (xs->rx && !xskq_empty_desc(xs->rx))
|
|
|
|
|
mask |= POLLIN | POLLRDNORM;
|
2018-05-02 19:01:34 +08:00
|
|
|
if (xs->tx && !xskq_full_desc(xs->tx))
|
|
|
|
|
mask |= POLLOUT | POLLWRNORM;
|
2018-05-02 19:01:27 +08:00
|
|
|
|
|
|
|
|
return mask;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:25 +08:00
|
|
|
static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
|
|
|
|
|
bool umem_queue)
|
2018-05-02 19:01:24 +08:00
|
|
|
{
|
|
|
|
|
struct xsk_queue *q;
|
|
|
|
|
|
|
|
|
|
if (entries == 0 || *queue || !is_power_of_2(entries))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
2018-05-02 19:01:25 +08:00
|
|
|
q = xskq_create(entries, umem_queue);
|
2018-05-02 19:01:24 +08:00
|
|
|
if (!q)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2018-05-22 15:35:01 +08:00
|
|
|
/* Make sure queue is ready before it can be seen by others */
|
|
|
|
|
smp_wmb();
|
2018-05-02 19:01:24 +08:00
|
|
|
*queue = q;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:23 +08:00
|
|
|
static int xsk_release(struct socket *sock)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk = sock->sk;
|
2018-05-02 19:01:26 +08:00
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2018-05-02 19:01:23 +08:00
|
|
|
struct net *net;
|
|
|
|
|
|
|
|
|
|
if (!sk)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
net = sock_net(sk);
|
|
|
|
|
|
|
|
|
|
local_bh_disable();
|
|
|
|
|
sock_prot_inuse_add(net, sk->sk_prot, -1);
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
|
2018-05-02 19:01:26 +08:00
|
|
|
if (xs->dev) {
|
2018-05-22 15:34:56 +08:00
|
|
|
/* Wait for driver to stop using the xdp socket. */
|
|
|
|
|
synchronize_net();
|
|
|
|
|
dev_put(xs->dev);
|
2018-05-02 19:01:26 +08:00
|
|
|
xs->dev = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:23 +08:00
|
|
|
sock_orphan(sk);
|
|
|
|
|
sock->sk = NULL;
|
|
|
|
|
|
|
|
|
|
sk_refcnt_debug_release(sk);
|
|
|
|
|
sock_put(sk);
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:26 +08:00
|
|
|
static struct socket *xsk_lookup_xsk_from_fd(int fd)
|
|
|
|
|
{
|
|
|
|
|
struct socket *sock;
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
sock = sockfd_lookup(fd, &err);
|
|
|
|
|
if (!sock)
|
|
|
|
|
return ERR_PTR(-ENOTSOCK);
|
|
|
|
|
|
|
|
|
|
if (sock->sk->sk_family != PF_XDP) {
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
return ERR_PTR(-ENOPROTOOPT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return sock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
|
|
|
|
|
{
|
|
|
|
|
struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
|
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2018-05-22 15:34:56 +08:00
|
|
|
struct net_device *dev;
|
2018-06-04 20:05:55 +08:00
|
|
|
u32 flags, qid;
|
2018-05-02 19:01:26 +08:00
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
|
|
if (addr_len < sizeof(struct sockaddr_xdp))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
if (sxdp->sxdp_family != AF_XDP)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-22 15:34:56 +08:00
|
|
|
if (xs->dev) {
|
|
|
|
|
err = -EBUSY;
|
|
|
|
|
goto out_release;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:26 +08:00
|
|
|
dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
|
|
|
|
|
if (!dev) {
|
|
|
|
|
err = -ENODEV;
|
|
|
|
|
goto out_release;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:32 +08:00
|
|
|
if (!xs->rx && !xs->tx) {
|
2018-05-02 19:01:26 +08:00
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
qid = sxdp->sxdp_queue_id;
|
|
|
|
|
|
|
|
|
|
if ((xs->rx && qid >= dev->real_num_rx_queues) ||
|
|
|
|
|
(xs->tx && qid >= dev->real_num_tx_queues)) {
|
2018-05-02 19:01:26 +08:00
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
flags = sxdp->sxdp_flags;
|
|
|
|
|
|
|
|
|
|
if (flags & XDP_SHARED_UMEM) {
|
2018-05-02 19:01:26 +08:00
|
|
|
struct xdp_sock *umem_xs;
|
|
|
|
|
struct socket *sock;
|
|
|
|
|
|
2018-06-04 20:05:55 +08:00
|
|
|
if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) {
|
|
|
|
|
/* Cannot specify flags for shared sockets. */
|
|
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:26 +08:00
|
|
|
if (xs->umem) {
|
|
|
|
|
/* We have already our own. */
|
|
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
|
|
|
|
|
if (IS_ERR(sock)) {
|
|
|
|
|
err = PTR_ERR(sock);
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
umem_xs = xdp_sk(sock->sk);
|
|
|
|
|
if (!umem_xs->umem) {
|
|
|
|
|
/* No umem to inherit. */
|
|
|
|
|
err = -EBADF;
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
goto out_unlock;
|
2018-06-04 20:05:55 +08:00
|
|
|
} else if (umem_xs->dev != dev || umem_xs->queue_id != qid) {
|
2018-05-02 19:01:26 +08:00
|
|
|
err = -EINVAL;
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
xdp_get_umem(umem_xs->umem);
|
|
|
|
|
xs->umem = umem_xs->umem;
|
|
|
|
|
sockfd_put(sock);
|
|
|
|
|
} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
|
|
|
|
|
err = -EINVAL;
|
|
|
|
|
goto out_unlock;
|
2018-05-02 19:01:27 +08:00
|
|
|
} else {
|
|
|
|
|
/* This xsk has its own umem. */
|
|
|
|
|
xskq_set_umem(xs->umem->fq, &xs->umem->props);
|
2018-05-02 19:01:31 +08:00
|
|
|
xskq_set_umem(xs->umem->cq, &xs->umem->props);
|
2018-06-04 20:05:55 +08:00
|
|
|
|
|
|
|
|
err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
|
|
|
|
|
if (err)
|
|
|
|
|
goto out_unlock;
|
2018-05-02 19:01:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
xs->dev = dev;
|
2018-06-04 20:05:57 +08:00
|
|
|
xs->zc = xs->umem->zc;
|
|
|
|
|
xs->queue_id = qid;
|
2018-05-02 19:01:26 +08:00
|
|
|
xskq_set_umem(xs->rx, &xs->umem->props);
|
2018-05-02 19:01:34 +08:00
|
|
|
xskq_set_umem(xs->tx, &xs->umem->props);
|
2018-06-04 20:05:57 +08:00
|
|
|
xdp_add_sk_umem(xs->umem, xs);
|
2018-05-02 19:01:26 +08:00
|
|
|
|
|
|
|
|
out_unlock:
|
|
|
|
|
if (err)
|
|
|
|
|
dev_put(dev);
|
|
|
|
|
out_release:
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:23 +08:00
|
|
|
static int xsk_setsockopt(struct socket *sock, int level, int optname,
|
|
|
|
|
char __user *optval, unsigned int optlen)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
if (level != SOL_XDP)
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
|
|
switch (optname) {
|
2018-05-02 19:01:25 +08:00
|
|
|
case XDP_RX_RING:
|
2018-05-02 19:01:32 +08:00
|
|
|
case XDP_TX_RING:
|
2018-05-02 19:01:25 +08:00
|
|
|
{
|
|
|
|
|
struct xsk_queue **q;
|
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
|
|
if (optlen < sizeof(entries))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
if (copy_from_user(&entries, optval, sizeof(entries)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-02 19:01:32 +08:00
|
|
|
q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
|
2018-05-02 19:01:25 +08:00
|
|
|
err = xsk_init_queue(entries, q, false);
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
2018-05-02 19:01:23 +08:00
|
|
|
case XDP_UMEM_REG:
|
|
|
|
|
{
|
|
|
|
|
struct xdp_umem_reg mr;
|
|
|
|
|
struct xdp_umem *umem;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(&mr, optval, sizeof(mr)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-22 15:35:02 +08:00
|
|
|
if (xs->umem) {
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return -EBUSY;
|
|
|
|
|
}
|
2018-05-02 19:01:23 +08:00
|
|
|
|
2018-05-22 15:35:02 +08:00
|
|
|
umem = xdp_umem_create(&mr);
|
|
|
|
|
if (IS_ERR(umem)) {
|
2018-05-02 19:01:23 +08:00
|
|
|
mutex_unlock(&xs->mutex);
|
2018-05-22 15:35:02 +08:00
|
|
|
return PTR_ERR(umem);
|
2018-05-02 19:01:23 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Make sure umem is ready before it can be seen by others */
|
|
|
|
|
smp_wmb();
|
|
|
|
|
xs->umem = umem;
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 19:01:24 +08:00
|
|
|
case XDP_UMEM_FILL_RING:
|
2018-05-02 19:01:31 +08:00
|
|
|
case XDP_UMEM_COMPLETION_RING:
|
2018-05-02 19:01:24 +08:00
|
|
|
{
|
|
|
|
|
struct xsk_queue **q;
|
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(&entries, optval, sizeof(entries)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2018-05-22 15:35:02 +08:00
|
|
|
if (!xs->umem) {
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:31 +08:00
|
|
|
q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
|
|
|
|
|
&xs->umem->cq;
|
2018-05-02 19:01:25 +08:00
|
|
|
err = xsk_init_queue(entries, q, true);
|
2018-05-02 19:01:24 +08:00
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
2018-05-02 19:01:23 +08:00
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:35 +08:00
|
|
|
static int xsk_getsockopt(struct socket *sock, int level, int optname,
|
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
int len;
|
|
|
|
|
|
|
|
|
|
if (level != SOL_XDP)
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
|
|
if (get_user(len, optlen))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (len < 0)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
switch (optname) {
|
|
|
|
|
case XDP_STATISTICS:
|
|
|
|
|
{
|
|
|
|
|
struct xdp_statistics stats;
|
|
|
|
|
|
|
|
|
|
if (len < sizeof(stats))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
|
stats.rx_dropped = xs->rx_dropped;
|
|
|
|
|
stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
|
|
|
|
|
stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
|
|
|
|
|
if (copy_to_user(optval, &stats, sizeof(stats)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (put_user(sizeof(stats), optlen))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 15:34:59 +08:00
|
|
|
case XDP_MMAP_OFFSETS:
|
|
|
|
|
{
|
|
|
|
|
struct xdp_mmap_offsets off;
|
|
|
|
|
|
|
|
|
|
if (len < sizeof(off))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
|
|
|
|
|
off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
|
|
|
|
|
off.rx.desc = offsetof(struct xdp_rxtx_ring, desc);
|
|
|
|
|
off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
|
|
|
|
|
off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
|
|
|
|
|
off.tx.desc = offsetof(struct xdp_rxtx_ring, desc);
|
|
|
|
|
|
|
|
|
|
off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
|
|
|
|
|
off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
|
|
|
|
|
off.fr.desc = offsetof(struct xdp_umem_ring, desc);
|
|
|
|
|
off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
|
|
|
|
|
off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
|
|
|
|
|
off.cr.desc = offsetof(struct xdp_umem_ring, desc);
|
|
|
|
|
|
|
|
|
|
len = sizeof(off);
|
|
|
|
|
if (copy_to_user(optval, &off, len))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (put_user(len, optlen))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2018-05-02 19:01:35 +08:00
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:24 +08:00
|
|
|
static int xsk_mmap(struct file *file, struct socket *sock,
|
|
|
|
|
struct vm_area_struct *vma)
|
|
|
|
|
{
|
2018-06-07 21:37:34 +08:00
|
|
|
loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
|
2018-05-02 19:01:24 +08:00
|
|
|
unsigned long size = vma->vm_end - vma->vm_start;
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sock->sk);
|
|
|
|
|
struct xsk_queue *q = NULL;
|
2018-05-22 15:35:01 +08:00
|
|
|
struct xdp_umem *umem;
|
2018-05-02 19:01:24 +08:00
|
|
|
unsigned long pfn;
|
|
|
|
|
struct page *qpg;
|
|
|
|
|
|
2018-05-02 19:01:25 +08:00
|
|
|
if (offset == XDP_PGOFF_RX_RING) {
|
2018-05-22 15:35:01 +08:00
|
|
|
q = READ_ONCE(xs->rx);
|
2018-05-02 19:01:32 +08:00
|
|
|
} else if (offset == XDP_PGOFF_TX_RING) {
|
2018-05-22 15:35:01 +08:00
|
|
|
q = READ_ONCE(xs->tx);
|
2018-05-02 19:01:25 +08:00
|
|
|
} else {
|
2018-05-22 15:35:01 +08:00
|
|
|
umem = READ_ONCE(xs->umem);
|
|
|
|
|
if (!umem)
|
2018-05-02 19:01:25 +08:00
|
|
|
return -EINVAL;
|
2018-05-02 19:01:24 +08:00
|
|
|
|
2018-05-02 19:01:25 +08:00
|
|
|
if (offset == XDP_UMEM_PGOFF_FILL_RING)
|
2018-05-22 15:35:01 +08:00
|
|
|
q = READ_ONCE(umem->fq);
|
2018-05-02 19:01:31 +08:00
|
|
|
else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
|
2018-05-22 15:35:01 +08:00
|
|
|
q = READ_ONCE(umem->cq);
|
2018-05-02 19:01:25 +08:00
|
|
|
}
|
2018-05-02 19:01:24 +08:00
|
|
|
|
|
|
|
|
if (!q)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
qpg = virt_to_head_page(q->ring);
|
|
|
|
|
if (size > (PAGE_SIZE << compound_order(qpg)))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
|
|
|
|
|
return remap_pfn_range(vma, vma->vm_start, pfn,
|
|
|
|
|
size, vma->vm_page_prot);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-02 19:01:23 +08:00
|
|
|
static struct proto xsk_proto = {
|
|
|
|
|
.name = "XDP",
|
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
.obj_size = sizeof(struct xdp_sock),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct proto_ops xsk_proto_ops = {
|
2018-05-18 20:00:24 +08:00
|
|
|
.family = PF_XDP,
|
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
.release = xsk_release,
|
|
|
|
|
.bind = xsk_bind,
|
|
|
|
|
.connect = sock_no_connect,
|
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
|
.accept = sock_no_accept,
|
|
|
|
|
.getname = sock_no_getname,
|
2018-06-29 00:43:44 +08:00
|
|
|
.poll = xsk_poll,
|
2018-05-18 20:00:24 +08:00
|
|
|
.ioctl = sock_no_ioctl,
|
|
|
|
|
.listen = sock_no_listen,
|
|
|
|
|
.shutdown = sock_no_shutdown,
|
|
|
|
|
.setsockopt = xsk_setsockopt,
|
|
|
|
|
.getsockopt = xsk_getsockopt,
|
|
|
|
|
.sendmsg = xsk_sendmsg,
|
|
|
|
|
.recvmsg = sock_no_recvmsg,
|
|
|
|
|
.mmap = xsk_mmap,
|
|
|
|
|
.sendpage = sock_no_sendpage,
|
2018-05-02 19:01:23 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static void xsk_destruct(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
|
return;
|
|
|
|
|
|
2018-05-02 19:01:25 +08:00
|
|
|
xskq_destroy(xs->rx);
|
2018-05-02 19:01:32 +08:00
|
|
|
xskq_destroy(xs->tx);
|
2018-06-04 20:05:57 +08:00
|
|
|
xdp_del_sk_umem(xs->umem, xs);
|
2018-05-02 19:01:23 +08:00
|
|
|
xdp_put_umem(xs->umem);
|
|
|
|
|
|
|
|
|
|
sk_refcnt_debug_dec(sk);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int xsk_create(struct net *net, struct socket *sock, int protocol,
|
|
|
|
|
int kern)
|
|
|
|
|
{
|
|
|
|
|
struct sock *sk;
|
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
|
|
if (!ns_capable(net->user_ns, CAP_NET_RAW))
|
|
|
|
|
return -EPERM;
|
|
|
|
|
if (sock->type != SOCK_RAW)
|
|
|
|
|
return -ESOCKTNOSUPPORT;
|
|
|
|
|
|
|
|
|
|
if (protocol)
|
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
|
|
|
|
|
|
sock->state = SS_UNCONNECTED;
|
|
|
|
|
|
|
|
|
|
sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
|
|
|
|
|
if (!sk)
|
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
|
|
sock->ops = &xsk_proto_ops;
|
|
|
|
|
|
|
|
|
|
sock_init_data(sock, sk);
|
|
|
|
|
|
|
|
|
|
sk->sk_family = PF_XDP;
|
|
|
|
|
|
|
|
|
|
sk->sk_destruct = xsk_destruct;
|
|
|
|
|
sk_refcnt_debug_inc(sk);
|
|
|
|
|
|
|
|
|
|
xs = xdp_sk(sk);
|
|
|
|
|
mutex_init(&xs->mutex);
|
|
|
|
|
|
|
|
|
|
local_bh_disable();
|
|
|
|
|
sock_prot_inuse_add(net, &xsk_proto, 1);
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct net_proto_family xsk_family_ops = {
|
|
|
|
|
.family = PF_XDP,
|
|
|
|
|
.create = xsk_create,
|
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int __init xsk_init(void)
|
|
|
|
|
{
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
err = proto_register(&xsk_proto, 0 /* no slab */);
|
|
|
|
|
if (err)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
err = sock_register(&xsk_family_ops);
|
|
|
|
|
if (err)
|
|
|
|
|
goto out_proto;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
out_proto:
|
|
|
|
|
proto_unregister(&xsk_proto);
|
|
|
|
|
out:
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fs_initcall(xsk_init);
|