2019-06-04 16:10:45 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2016-11-23 23:52:26 +08:00
|
|
|
/*
|
|
|
|
|
* Functions to manage eBPF programs attached to cgroups
|
|
|
|
|
*
|
|
|
|
|
* Copyright (c) 2016 Daniel Mack
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
|
#include <linux/atomic.h>
|
|
|
|
|
#include <linux/cgroup.h>
|
2019-02-28 04:59:24 +08:00
|
|
|
#include <linux/filter.h>
|
2016-11-23 23:52:26 +08:00
|
|
|
#include <linux/slab.h>
|
2019-02-28 04:59:24 +08:00
|
|
|
#include <linux/sysctl.h>
|
2019-02-28 05:28:48 +08:00
|
|
|
#include <linux/string.h>
|
2016-11-23 23:52:26 +08:00
|
|
|
#include <linux/bpf.h>
|
|
|
|
|
#include <linux/bpf-cgroup.h>
|
|
|
|
|
#include <net/sock.h>
|
bpf: implement getsockopt and setsockopt hooks
Implement new BPF_PROG_TYPE_CGROUP_SOCKOPT program type and
BPF_CGROUP_{G,S}ETSOCKOPT cgroup hooks.
BPF_CGROUP_SETSOCKOPT can modify user setsockopt arguments before
passing them down to the kernel or bypass kernel completely.
BPF_CGROUP_GETSOCKOPT can can inspect/modify getsockopt arguments that
kernel returns.
Both hooks reuse existing PTR_TO_PACKET{,_END} infrastructure.
The buffer memory is pre-allocated (because I don't think there is
a precedent for working with __user memory from bpf). This might be
slow to do for each {s,g}etsockopt call, that's why I've added
__cgroup_bpf_prog_array_is_empty that exits early if there is nothing
attached to a cgroup. Note, however, that there is a race between
__cgroup_bpf_prog_array_is_empty and BPF_PROG_RUN_ARRAY where cgroup
program layout might have changed; this should not be a problem
because in general there is a race between multiple calls to
{s,g}etsocktop and user adding/removing bpf progs from a cgroup.
The return code of the BPF program is handled as follows:
* 0: EPERM
* 1: success, continue with next BPF program in the cgroup chain
v9:
* allow overwriting setsockopt arguments (Alexei Starovoitov):
* use set_fs (same as kernel_setsockopt)
* buffer is always kzalloc'd (no small on-stack buffer)
v8:
* use s32 for optlen (Andrii Nakryiko)
v7:
* return only 0 or 1 (Alexei Starovoitov)
* always run all progs (Alexei Starovoitov)
* use optval=0 as kernel bypass in setsockopt (Alexei Starovoitov)
(decided to use optval=-1 instead, optval=0 might be a valid input)
* call getsockopt hook after kernel handlers (Alexei Starovoitov)
v6:
* rework cgroup chaining; stop as soon as bpf program returns
0 or 2; see patch with the documentation for the details
* drop Andrii's and Martin's Acked-by (not sure they are comfortable
with the new state of things)
v5:
* skip copy_to_user() and put_user() when ret == 0 (Martin Lau)
v4:
* don't export bpf_sk_fullsock helper (Martin Lau)
* size != sizeof(__u64) for uapi pointers (Martin Lau)
* offsetof instead of bpf_ctx_range when checking ctx access (Martin Lau)
v3:
* typos in BPF_PROG_CGROUP_SOCKOPT_RUN_ARRAY comments (Andrii Nakryiko)
* reverse christmas tree in BPF_PROG_CGROUP_SOCKOPT_RUN_ARRAY (Andrii
Nakryiko)
* use __bpf_md_ptr instead of __u32 for optval{,_end} (Martin Lau)
* use BPF_FIELD_SIZEOF() for consistency (Martin Lau)
* new CG_SOCKOPT_ACCESS macro to wrap repeated parts
v2:
* moved bpf_sockopt_kern fields around to remove a hole (Martin Lau)
* aligned bpf_sockopt_kern->buf to 8 bytes (Martin Lau)
* bpf_prog_array_is_empty instead of bpf_prog_array_length (Martin Lau)
* added [0,2] return code check to verifier (Martin Lau)
* dropped unused buf[64] from the stack (Martin Lau)
* use PTR_TO_SOCKET for bpf_sockopt->sk (Martin Lau)
* dropped bpf_target_off from ctx rewrites (Martin Lau)
* use return code for kernel bypass (Martin Lau & Andrii Nakryiko)
Cc: Andrii Nakryiko <andriin@fb.com>
Cc: Martin Lau <kafai@fb.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-06-28 04:38:47 +08:00
|
|
|
#include <net/bpf_sk_storage.h>
|
2016-11-23 23:52:26 +08:00
|
|
|
|
bpf: fix cgroup bpf release synchronization
Since commit 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf
from cgroup itself"), cgroup_bpf release occurs asynchronously
(from a worker context), and before the release of the cgroup itself.
This introduced a previously non-existing race between the release
and update paths. E.g. if a leaf's cgroup_bpf is released and a new
bpf program is attached to the one of ancestor cgroups at the same
time. The race may result in double-free and other memory corruptions.
To fix the problem, let's protect the body of cgroup_bpf_release()
with cgroup_mutex, as it was effectively previously, when all this
code was called from the cgroup release path with cgroup mutex held.
Also let's skip cgroups, which have no chances to invoke a bpf
program, on the update path. If the cgroup bpf refcnt reached 0,
it means that the cgroup is offline (no attached processes), and
there are no associated sockets left. It means there is no point
in updating effective progs array! And it can lead to a leak,
if it happens after the release. So, let's skip such cgroups.
Big thanks for Tejun Heo for discovering and debugging of this problem!
Fixes: 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf from cgroup itself")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-06-26 05:38:58 +08:00
|
|
|
#include "../cgroup/cgroup-internal.h"
|
|
|
|
|
|
2016-11-23 23:52:26 +08:00
|
|
|
DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
|
|
|
|
|
EXPORT_SYMBOL(cgroup_bpf_enabled_key);
|
|
|
|
|
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
void cgroup_bpf_offline(struct cgroup *cgrp)
|
|
|
|
|
{
|
|
|
|
|
cgroup_get(cgrp);
|
|
|
|
|
percpu_ref_kill(&cgrp->bpf.refcnt);
|
|
|
|
|
}
|
|
|
|
|
|
2016-11-23 23:52:26 +08:00
|
|
|
/**
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
* cgroup_bpf_release() - put references of all bpf programs and
|
|
|
|
|
* release all cgroup bpf data
|
|
|
|
|
* @work: work structure embedded into the cgroup to modify
|
2016-11-23 23:52:26 +08:00
|
|
|
*/
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
static void cgroup_bpf_release(struct work_struct *work)
|
2016-11-23 23:52:26 +08:00
|
|
|
{
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
struct cgroup *cgrp = container_of(work, struct cgroup,
|
|
|
|
|
bpf.release_work);
|
2018-09-28 22:45:36 +08:00
|
|
|
enum bpf_cgroup_storage_type stype;
|
2019-05-29 05:14:43 +08:00
|
|
|
struct bpf_prog_array *old_array;
|
2016-11-23 23:52:26 +08:00
|
|
|
unsigned int type;
|
|
|
|
|
|
bpf: fix cgroup bpf release synchronization
Since commit 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf
from cgroup itself"), cgroup_bpf release occurs asynchronously
(from a worker context), and before the release of the cgroup itself.
This introduced a previously non-existing race between the release
and update paths. E.g. if a leaf's cgroup_bpf is released and a new
bpf program is attached to the one of ancestor cgroups at the same
time. The race may result in double-free and other memory corruptions.
To fix the problem, let's protect the body of cgroup_bpf_release()
with cgroup_mutex, as it was effectively previously, when all this
code was called from the cgroup release path with cgroup mutex held.
Also let's skip cgroups, which have no chances to invoke a bpf
program, on the update path. If the cgroup bpf refcnt reached 0,
it means that the cgroup is offline (no attached processes), and
there are no associated sockets left. It means there is no point
in updating effective progs array! And it can lead to a leak,
if it happens after the release. So, let's skip such cgroups.
Big thanks for Tejun Heo for discovering and debugging of this problem!
Fixes: 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf from cgroup itself")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-06-26 05:38:58 +08:00
|
|
|
mutex_lock(&cgroup_mutex);
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
|
|
|
|
|
struct list_head *progs = &cgrp->bpf.progs[type];
|
|
|
|
|
struct bpf_prog_list *pl, *tmp;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry_safe(pl, tmp, progs, node) {
|
|
|
|
|
list_del(&pl->node);
|
|
|
|
|
bpf_prog_put(pl->prog);
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype) {
|
|
|
|
|
bpf_cgroup_storage_unlink(pl->storage[stype]);
|
|
|
|
|
bpf_cgroup_storage_free(pl->storage[stype]);
|
|
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
kfree(pl);
|
2016-11-23 23:52:26 +08:00
|
|
|
static_branch_dec(&cgroup_bpf_enabled_key);
|
|
|
|
|
}
|
2019-05-29 05:14:43 +08:00
|
|
|
old_array = rcu_dereference_protected(
|
|
|
|
|
cgrp->bpf.effective[type],
|
bpf: fix cgroup bpf release synchronization
Since commit 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf
from cgroup itself"), cgroup_bpf release occurs asynchronously
(from a worker context), and before the release of the cgroup itself.
This introduced a previously non-existing race between the release
and update paths. E.g. if a leaf's cgroup_bpf is released and a new
bpf program is attached to the one of ancestor cgroups at the same
time. The race may result in double-free and other memory corruptions.
To fix the problem, let's protect the body of cgroup_bpf_release()
with cgroup_mutex, as it was effectively previously, when all this
code was called from the cgroup release path with cgroup mutex held.
Also let's skip cgroups, which have no chances to invoke a bpf
program, on the update path. If the cgroup bpf refcnt reached 0,
it means that the cgroup is offline (no attached processes), and
there are no associated sockets left. It means there is no point
in updating effective progs array! And it can lead to a leak,
if it happens after the release. So, let's skip such cgroups.
Big thanks for Tejun Heo for discovering and debugging of this problem!
Fixes: 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf from cgroup itself")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-06-26 05:38:58 +08:00
|
|
|
lockdep_is_held(&cgroup_mutex));
|
2019-05-29 05:14:43 +08:00
|
|
|
bpf_prog_array_free(old_array);
|
2017-10-03 13:50:21 +08:00
|
|
|
}
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
|
bpf: fix cgroup bpf release synchronization
Since commit 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf
from cgroup itself"), cgroup_bpf release occurs asynchronously
(from a worker context), and before the release of the cgroup itself.
This introduced a previously non-existing race between the release
and update paths. E.g. if a leaf's cgroup_bpf is released and a new
bpf program is attached to the one of ancestor cgroups at the same
time. The race may result in double-free and other memory corruptions.
To fix the problem, let's protect the body of cgroup_bpf_release()
with cgroup_mutex, as it was effectively previously, when all this
code was called from the cgroup release path with cgroup mutex held.
Also let's skip cgroups, which have no chances to invoke a bpf
program, on the update path. If the cgroup bpf refcnt reached 0,
it means that the cgroup is offline (no attached processes), and
there are no associated sockets left. It means there is no point
in updating effective progs array! And it can lead to a leak,
if it happens after the release. So, let's skip such cgroups.
Big thanks for Tejun Heo for discovering and debugging of this problem!
Fixes: 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf from cgroup itself")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-06-26 05:38:58 +08:00
|
|
|
mutex_unlock(&cgroup_mutex);
|
|
|
|
|
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
percpu_ref_exit(&cgrp->bpf.refcnt);
|
|
|
|
|
cgroup_put(cgrp);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* cgroup_bpf_release_fn() - callback used to schedule releasing
|
|
|
|
|
* of bpf cgroup data
|
|
|
|
|
* @ref: percpu ref counter structure
|
|
|
|
|
*/
|
|
|
|
|
static void cgroup_bpf_release_fn(struct percpu_ref *ref)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);
|
|
|
|
|
|
|
|
|
|
INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);
|
|
|
|
|
queue_work(system_wq, &cgrp->bpf.release_work);
|
2017-10-03 13:50:21 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* count number of elements in the list.
|
|
|
|
|
* it's slow but the list cannot be long
|
|
|
|
|
*/
|
|
|
|
|
static u32 prog_list_length(struct list_head *head)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_prog_list *pl;
|
|
|
|
|
u32 cnt = 0;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(pl, head, node) {
|
|
|
|
|
if (!pl->prog)
|
|
|
|
|
continue;
|
|
|
|
|
cnt++;
|
2016-11-23 23:52:26 +08:00
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
return cnt;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* if parent has non-overridable prog attached,
|
|
|
|
|
* disallow attaching new programs to the descendent cgroup.
|
|
|
|
|
* if parent has overridable or multi-prog, allow attaching
|
|
|
|
|
*/
|
|
|
|
|
static bool hierarchy_allows_attach(struct cgroup *cgrp,
|
|
|
|
|
enum bpf_attach_type type,
|
|
|
|
|
u32 new_flags)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *p;
|
|
|
|
|
|
|
|
|
|
p = cgroup_parent(cgrp);
|
|
|
|
|
if (!p)
|
|
|
|
|
return true;
|
|
|
|
|
do {
|
|
|
|
|
u32 flags = p->bpf.flags[type];
|
|
|
|
|
u32 cnt;
|
|
|
|
|
|
|
|
|
|
if (flags & BPF_F_ALLOW_MULTI)
|
|
|
|
|
return true;
|
|
|
|
|
cnt = prog_list_length(&p->bpf.progs[type]);
|
|
|
|
|
WARN_ON_ONCE(cnt > 1);
|
|
|
|
|
if (cnt == 1)
|
|
|
|
|
return !!(flags & BPF_F_ALLOW_OVERRIDE);
|
|
|
|
|
p = cgroup_parent(p);
|
|
|
|
|
} while (p);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* compute a chain of effective programs for a given cgroup:
|
|
|
|
|
* start from the list of programs in this cgroup and add
|
|
|
|
|
* all parent programs.
|
|
|
|
|
* Note that parent's F_ALLOW_OVERRIDE-type program is yielding
|
|
|
|
|
* to programs in this cgroup
|
|
|
|
|
*/
|
|
|
|
|
static int compute_effective_progs(struct cgroup *cgrp,
|
|
|
|
|
enum bpf_attach_type type,
|
2019-05-29 05:14:43 +08:00
|
|
|
struct bpf_prog_array **array)
|
2017-10-03 13:50:21 +08:00
|
|
|
{
|
2018-09-28 22:45:36 +08:00
|
|
|
enum bpf_cgroup_storage_type stype;
|
2018-07-14 03:41:11 +08:00
|
|
|
struct bpf_prog_array *progs;
|
2017-10-03 13:50:21 +08:00
|
|
|
struct bpf_prog_list *pl;
|
|
|
|
|
struct cgroup *p = cgrp;
|
|
|
|
|
int cnt = 0;
|
|
|
|
|
|
|
|
|
|
/* count number of effective programs by walking parents */
|
|
|
|
|
do {
|
|
|
|
|
if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
|
|
|
|
|
cnt += prog_list_length(&p->bpf.progs[type]);
|
|
|
|
|
p = cgroup_parent(p);
|
|
|
|
|
} while (p);
|
|
|
|
|
|
|
|
|
|
progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);
|
|
|
|
|
if (!progs)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* populate the array with effective progs */
|
|
|
|
|
cnt = 0;
|
|
|
|
|
p = cgrp;
|
|
|
|
|
do {
|
2018-08-03 05:27:21 +08:00
|
|
|
if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(pl, &p->bpf.progs[type], node) {
|
|
|
|
|
if (!pl->prog)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
progs->items[cnt].prog = pl->prog;
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
progs->items[cnt].cgroup_storage[stype] =
|
|
|
|
|
pl->storage[stype];
|
2018-08-03 05:27:21 +08:00
|
|
|
cnt++;
|
|
|
|
|
}
|
|
|
|
|
} while ((p = cgroup_parent(p)));
|
2017-10-03 13:50:21 +08:00
|
|
|
|
2019-05-29 05:14:43 +08:00
|
|
|
*array = progs;
|
2017-10-03 13:50:21 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void activate_effective_progs(struct cgroup *cgrp,
|
|
|
|
|
enum bpf_attach_type type,
|
2019-05-29 05:14:43 +08:00
|
|
|
struct bpf_prog_array *old_array)
|
2017-10-03 13:50:21 +08:00
|
|
|
{
|
2019-05-29 05:14:43 +08:00
|
|
|
rcu_swap_protected(cgrp->bpf.effective[type], old_array,
|
|
|
|
|
lockdep_is_held(&cgroup_mutex));
|
2017-10-03 13:50:21 +08:00
|
|
|
/* free prog array after grace period, since __cgroup_bpf_run_*()
|
|
|
|
|
* might be still walking the array
|
|
|
|
|
*/
|
|
|
|
|
bpf_prog_array_free(old_array);
|
2016-11-23 23:52:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* cgroup_bpf_inherit() - inherit effective programs from parent
|
|
|
|
|
* @cgrp: the cgroup to modify
|
|
|
|
|
*/
|
2017-10-03 13:50:21 +08:00
|
|
|
int cgroup_bpf_inherit(struct cgroup *cgrp)
|
2016-11-23 23:52:26 +08:00
|
|
|
{
|
2017-10-03 13:50:21 +08:00
|
|
|
/* has to use marco instead of const int, since compiler thinks
|
|
|
|
|
* that array below is variable length
|
|
|
|
|
*/
|
|
|
|
|
#define NR ARRAY_SIZE(cgrp->bpf.effective)
|
2019-05-29 05:14:43 +08:00
|
|
|
struct bpf_prog_array *arrays[NR] = {};
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
int ret, i;
|
|
|
|
|
|
|
|
|
|
ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
|
|
|
|
|
GFP_KERNEL);
|
|
|
|
|
if (ret)
|
|
|
|
|
return ret;
|
2016-11-23 23:52:26 +08:00
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
for (i = 0; i < NR; i++)
|
|
|
|
|
INIT_LIST_HEAD(&cgrp->bpf.progs[i]);
|
2016-11-23 23:52:26 +08:00
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
for (i = 0; i < NR; i++)
|
|
|
|
|
if (compute_effective_progs(cgrp, i, &arrays[i]))
|
|
|
|
|
goto cleanup;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < NR; i++)
|
|
|
|
|
activate_effective_progs(cgrp, i, arrays[i]);
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
cleanup:
|
|
|
|
|
for (i = 0; i < NR; i++)
|
|
|
|
|
bpf_prog_array_free(arrays[i]);
|
bpf: decouple the lifetime of cgroup_bpf from cgroup itself
Currently the lifetime of bpf programs attached to a cgroup is bound
to the lifetime of the cgroup itself. It means that if a user
forgets (or intentionally avoids) to detach a bpf program before
removing the cgroup, it will stay attached up to the release of the
cgroup. Since the cgroup can stay in the dying state (the state
between being rmdir()'ed and being released) for a very long time, it
leads to a waste of memory. Also, it blocks a possibility to implement
the memcg-based memory accounting for bpf objects, because a circular
reference dependency will occur. Charged memory pages are pinning the
corresponding memory cgroup, and if the memory cgroup is pinning
the attached bpf program, nothing will be ever released.
A dying cgroup can not contain any processes, so the only chance for
an attached bpf program to be executed is a live socket associated
with the cgroup. So in order to release all bpf data early, let's
count associated sockets using a new percpu refcounter. On cgroup
removal the counter is transitioned to the atomic mode, and as soon
as it reaches 0, all bpf programs are detached.
Because cgroup_bpf_release() can block, it can't be called from
the percpu ref counter callback directly, so instead an asynchronous
work is scheduled.
The reference counter is not socket specific, and can be used for any
other types of programs, which can be executed from a cgroup-bpf hook
outside of the process context, had such a need arise in the future.
Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: jolsa@redhat.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-05-26 00:37:39 +08:00
|
|
|
|
|
|
|
|
percpu_ref_exit(&cgrp->bpf.refcnt);
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
return -ENOMEM;
|
2016-11-23 23:52:26 +08:00
|
|
|
}
|
|
|
|
|
|
2018-08-07 05:27:28 +08:00
|
|
|
static int update_effective_progs(struct cgroup *cgrp,
|
|
|
|
|
enum bpf_attach_type type)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup_subsys_state *css;
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
/* allocate and recompute effective prog arrays */
|
|
|
|
|
css_for_each_descendant_pre(css, &cgrp->self) {
|
|
|
|
|
struct cgroup *desc = container_of(css, struct cgroup, self);
|
|
|
|
|
|
bpf: fix cgroup bpf release synchronization
Since commit 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf
from cgroup itself"), cgroup_bpf release occurs asynchronously
(from a worker context), and before the release of the cgroup itself.
This introduced a previously non-existing race between the release
and update paths. E.g. if a leaf's cgroup_bpf is released and a new
bpf program is attached to the one of ancestor cgroups at the same
time. The race may result in double-free and other memory corruptions.
To fix the problem, let's protect the body of cgroup_bpf_release()
with cgroup_mutex, as it was effectively previously, when all this
code was called from the cgroup release path with cgroup mutex held.
Also let's skip cgroups, which have no chances to invoke a bpf
program, on the update path. If the cgroup bpf refcnt reached 0,
it means that the cgroup is offline (no attached processes), and
there are no associated sockets left. It means there is no point
in updating effective progs array! And it can lead to a leak,
if it happens after the release. So, let's skip such cgroups.
Big thanks for Tejun Heo for discovering and debugging of this problem!
Fixes: 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf from cgroup itself")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-06-26 05:38:58 +08:00
|
|
|
if (percpu_ref_is_zero(&desc->bpf.refcnt))
|
|
|
|
|
continue;
|
|
|
|
|
|
2018-08-07 05:27:28 +08:00
|
|
|
err = compute_effective_progs(desc, type, &desc->bpf.inactive);
|
|
|
|
|
if (err)
|
|
|
|
|
goto cleanup;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* all allocations were successful. Activate all prog arrays */
|
|
|
|
|
css_for_each_descendant_pre(css, &cgrp->self) {
|
|
|
|
|
struct cgroup *desc = container_of(css, struct cgroup, self);
|
|
|
|
|
|
bpf: fix cgroup bpf release synchronization
Since commit 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf
from cgroup itself"), cgroup_bpf release occurs asynchronously
(from a worker context), and before the release of the cgroup itself.
This introduced a previously non-existing race between the release
and update paths. E.g. if a leaf's cgroup_bpf is released and a new
bpf program is attached to the one of ancestor cgroups at the same
time. The race may result in double-free and other memory corruptions.
To fix the problem, let's protect the body of cgroup_bpf_release()
with cgroup_mutex, as it was effectively previously, when all this
code was called from the cgroup release path with cgroup mutex held.
Also let's skip cgroups, which have no chances to invoke a bpf
program, on the update path. If the cgroup bpf refcnt reached 0,
it means that the cgroup is offline (no attached processes), and
there are no associated sockets left. It means there is no point
in updating effective progs array! And it can lead to a leak,
if it happens after the release. So, let's skip such cgroups.
Big thanks for Tejun Heo for discovering and debugging of this problem!
Fixes: 4bfc0bb2c60e ("bpf: decouple the lifetime of cgroup_bpf from cgroup itself")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-06-26 05:38:58 +08:00
|
|
|
if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
|
|
|
|
|
if (unlikely(desc->bpf.inactive)) {
|
|
|
|
|
bpf_prog_array_free(desc->bpf.inactive);
|
|
|
|
|
desc->bpf.inactive = NULL;
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2018-08-07 05:27:28 +08:00
|
|
|
activate_effective_progs(desc, type, desc->bpf.inactive);
|
|
|
|
|
desc->bpf.inactive = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
cleanup:
|
|
|
|
|
/* oom while computing effective. Free all computed effective arrays
|
|
|
|
|
* since they were not activated
|
|
|
|
|
*/
|
|
|
|
|
css_for_each_descendant_pre(css, &cgrp->self) {
|
|
|
|
|
struct cgroup *desc = container_of(css, struct cgroup, self);
|
|
|
|
|
|
|
|
|
|
bpf_prog_array_free(desc->bpf.inactive);
|
|
|
|
|
desc->bpf.inactive = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
#define BPF_CGROUP_MAX_PROGS 64
|
|
|
|
|
|
2016-11-23 23:52:26 +08:00
|
|
|
/**
|
2017-10-03 13:50:21 +08:00
|
|
|
* __cgroup_bpf_attach() - Attach the program to a cgroup, and
|
2016-11-23 23:52:26 +08:00
|
|
|
* propagate the change to descendants
|
|
|
|
|
* @cgrp: The cgroup which descendants to traverse
|
2017-10-03 13:50:21 +08:00
|
|
|
* @prog: A program to attach
|
|
|
|
|
* @type: Type of attach operation
|
2019-01-29 14:47:06 +08:00
|
|
|
* @flags: Option flags
|
2016-11-23 23:52:26 +08:00
|
|
|
*
|
|
|
|
|
* Must be called with cgroup_mutex held.
|
|
|
|
|
*/
|
2017-10-03 13:50:21 +08:00
|
|
|
int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
|
|
|
|
|
enum bpf_attach_type type, u32 flags)
|
2016-11-23 23:52:26 +08:00
|
|
|
{
|
2017-10-03 13:50:21 +08:00
|
|
|
struct list_head *progs = &cgrp->bpf.progs[type];
|
|
|
|
|
struct bpf_prog *old_prog = NULL;
|
2018-09-28 22:45:36 +08:00
|
|
|
struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE],
|
|
|
|
|
*old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {NULL};
|
|
|
|
|
enum bpf_cgroup_storage_type stype;
|
2017-10-03 13:50:21 +08:00
|
|
|
struct bpf_prog_list *pl;
|
|
|
|
|
bool pl_was_allocated;
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))
|
|
|
|
|
/* invalid combination */
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
if (!hierarchy_allows_attach(cgrp, type, flags))
|
2017-02-11 12:28:24 +08:00
|
|
|
return -EPERM;
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
if (!list_empty(progs) && cgrp->bpf.flags[type] != flags)
|
|
|
|
|
/* Disallow attaching non-overridable on top
|
|
|
|
|
* of existing overridable in this cgroup.
|
|
|
|
|
* Disallow attaching multi-prog if overridable or none
|
2017-02-11 12:28:24 +08:00
|
|
|
*/
|
|
|
|
|
return -EPERM;
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
|
|
|
|
|
return -E2BIG;
|
|
|
|
|
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype) {
|
|
|
|
|
storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
|
|
|
|
|
if (IS_ERR(storage[stype])) {
|
|
|
|
|
storage[stype] = NULL;
|
|
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
bpf_cgroup_storage_free(storage[stype]);
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
}
|
2018-08-03 05:27:20 +08:00
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
if (flags & BPF_F_ALLOW_MULTI) {
|
2018-08-03 05:27:20 +08:00
|
|
|
list_for_each_entry(pl, progs, node) {
|
|
|
|
|
if (pl->prog == prog) {
|
2017-10-03 13:50:21 +08:00
|
|
|
/* disallow attaching the same prog twice */
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
bpf_cgroup_storage_free(storage[stype]);
|
2017-10-03 13:50:21 +08:00
|
|
|
return -EINVAL;
|
2018-08-03 05:27:20 +08:00
|
|
|
}
|
|
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
|
|
|
|
|
pl = kmalloc(sizeof(*pl), GFP_KERNEL);
|
2018-08-03 05:27:20 +08:00
|
|
|
if (!pl) {
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
bpf_cgroup_storage_free(storage[stype]);
|
2017-10-03 13:50:21 +08:00
|
|
|
return -ENOMEM;
|
2018-08-03 05:27:20 +08:00
|
|
|
}
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
pl_was_allocated = true;
|
|
|
|
|
pl->prog = prog;
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
pl->storage[stype] = storage[stype];
|
2017-10-03 13:50:21 +08:00
|
|
|
list_add_tail(&pl->node, progs);
|
|
|
|
|
} else {
|
|
|
|
|
if (list_empty(progs)) {
|
|
|
|
|
pl = kmalloc(sizeof(*pl), GFP_KERNEL);
|
2018-08-03 05:27:20 +08:00
|
|
|
if (!pl) {
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
bpf_cgroup_storage_free(storage[stype]);
|
2017-10-03 13:50:21 +08:00
|
|
|
return -ENOMEM;
|
2018-08-03 05:27:20 +08:00
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
pl_was_allocated = true;
|
|
|
|
|
list_add_tail(&pl->node, progs);
|
|
|
|
|
} else {
|
|
|
|
|
pl = list_first_entry(progs, typeof(*pl), node);
|
|
|
|
|
old_prog = pl->prog;
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype) {
|
|
|
|
|
old_storage[stype] = pl->storage[stype];
|
|
|
|
|
bpf_cgroup_storage_unlink(old_storage[stype]);
|
|
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
pl_was_allocated = false;
|
|
|
|
|
}
|
|
|
|
|
pl->prog = prog;
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
pl->storage[stype] = storage[stype];
|
2017-02-11 12:28:24 +08:00
|
|
|
}
|
2016-11-23 23:52:26 +08:00
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
cgrp->bpf.flags[type] = flags;
|
2017-02-11 12:28:24 +08:00
|
|
|
|
2018-08-07 05:27:28 +08:00
|
|
|
err = update_effective_progs(cgrp, type);
|
|
|
|
|
if (err)
|
|
|
|
|
goto cleanup;
|
2017-10-03 13:50:21 +08:00
|
|
|
|
|
|
|
|
static_branch_inc(&cgroup_bpf_enabled_key);
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype) {
|
|
|
|
|
if (!old_storage[stype])
|
|
|
|
|
continue;
|
|
|
|
|
bpf_cgroup_storage_free(old_storage[stype]);
|
|
|
|
|
}
|
2016-11-23 23:52:26 +08:00
|
|
|
if (old_prog) {
|
|
|
|
|
bpf_prog_put(old_prog);
|
|
|
|
|
static_branch_dec(&cgroup_bpf_enabled_key);
|
|
|
|
|
}
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype)
|
|
|
|
|
bpf_cgroup_storage_link(storage[stype], cgrp, type);
|
2017-02-11 12:28:24 +08:00
|
|
|
return 0;
|
2017-10-03 13:50:21 +08:00
|
|
|
|
|
|
|
|
cleanup:
|
|
|
|
|
/* and cleanup the prog list */
|
|
|
|
|
pl->prog = old_prog;
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype) {
|
|
|
|
|
bpf_cgroup_storage_free(pl->storage[stype]);
|
|
|
|
|
pl->storage[stype] = old_storage[stype];
|
|
|
|
|
bpf_cgroup_storage_link(old_storage[stype], cgrp, type);
|
|
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
if (pl_was_allocated) {
|
|
|
|
|
list_del(&pl->node);
|
|
|
|
|
kfree(pl);
|
|
|
|
|
}
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* __cgroup_bpf_detach() - Detach the program from a cgroup, and
|
|
|
|
|
* propagate the change to descendants
|
|
|
|
|
* @cgrp: The cgroup which descendants to traverse
|
|
|
|
|
* @prog: A program to detach or NULL
|
|
|
|
|
* @type: Type of detach operation
|
|
|
|
|
*
|
|
|
|
|
* Must be called with cgroup_mutex held.
|
|
|
|
|
*/
|
|
|
|
|
int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
|
2019-01-29 14:47:06 +08:00
|
|
|
enum bpf_attach_type type)
|
2017-10-03 13:50:21 +08:00
|
|
|
{
|
|
|
|
|
struct list_head *progs = &cgrp->bpf.progs[type];
|
2018-09-28 22:45:36 +08:00
|
|
|
enum bpf_cgroup_storage_type stype;
|
2017-10-03 13:50:21 +08:00
|
|
|
u32 flags = cgrp->bpf.flags[type];
|
|
|
|
|
struct bpf_prog *old_prog = NULL;
|
|
|
|
|
struct bpf_prog_list *pl;
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
if (flags & BPF_F_ALLOW_MULTI) {
|
|
|
|
|
if (!prog)
|
|
|
|
|
/* to detach MULTI prog the user has to specify valid FD
|
|
|
|
|
* of the program to be detached
|
|
|
|
|
*/
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
} else {
|
|
|
|
|
if (list_empty(progs))
|
|
|
|
|
/* report error when trying to detach and nothing is attached */
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (flags & BPF_F_ALLOW_MULTI) {
|
|
|
|
|
/* find the prog and detach it */
|
|
|
|
|
list_for_each_entry(pl, progs, node) {
|
|
|
|
|
if (pl->prog != prog)
|
|
|
|
|
continue;
|
|
|
|
|
old_prog = prog;
|
|
|
|
|
/* mark it deleted, so it's ignored while
|
|
|
|
|
* recomputing effective
|
|
|
|
|
*/
|
|
|
|
|
pl->prog = NULL;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
if (!old_prog)
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
} else {
|
|
|
|
|
/* to maintain backward compatibility NONE and OVERRIDE cgroups
|
|
|
|
|
* allow detaching with invalid FD (prog==NULL)
|
|
|
|
|
*/
|
|
|
|
|
pl = list_first_entry(progs, typeof(*pl), node);
|
|
|
|
|
old_prog = pl->prog;
|
|
|
|
|
pl->prog = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2018-08-07 05:27:28 +08:00
|
|
|
err = update_effective_progs(cgrp, type);
|
|
|
|
|
if (err)
|
|
|
|
|
goto cleanup;
|
2017-10-03 13:50:21 +08:00
|
|
|
|
|
|
|
|
/* now can actually delete it from this cgroup list */
|
|
|
|
|
list_del(&pl->node);
|
2018-09-28 22:45:36 +08:00
|
|
|
for_each_cgroup_storage_type(stype) {
|
|
|
|
|
bpf_cgroup_storage_unlink(pl->storage[stype]);
|
|
|
|
|
bpf_cgroup_storage_free(pl->storage[stype]);
|
|
|
|
|
}
|
2017-10-03 13:50:21 +08:00
|
|
|
kfree(pl);
|
|
|
|
|
if (list_empty(progs))
|
|
|
|
|
/* last program was detached, reset flags to zero */
|
|
|
|
|
cgrp->bpf.flags[type] = 0;
|
|
|
|
|
|
|
|
|
|
bpf_prog_put(old_prog);
|
|
|
|
|
static_branch_dec(&cgroup_bpf_enabled_key);
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
cleanup:
|
|
|
|
|
/* and restore back old_prog */
|
|
|
|
|
pl->prog = old_prog;
|
|
|
|
|
return err;
|
2016-11-23 23:52:26 +08:00
|
|
|
}
|
|
|
|
|
|
2017-10-03 13:50:22 +08:00
|
|
|
/* Must be called with cgroup_mutex held to avoid races. */
|
|
|
|
|
int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
|
|
|
|
|
union bpf_attr __user *uattr)
|
|
|
|
|
{
|
|
|
|
|
__u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
|
|
|
|
|
enum bpf_attach_type type = attr->query.attach_type;
|
|
|
|
|
struct list_head *progs = &cgrp->bpf.progs[type];
|
|
|
|
|
u32 flags = cgrp->bpf.flags[type];
|
2019-05-29 05:14:43 +08:00
|
|
|
struct bpf_prog_array *effective;
|
2017-10-03 13:50:22 +08:00
|
|
|
int cnt, ret = 0, i;
|
|
|
|
|
|
2019-05-29 05:14:43 +08:00
|
|
|
effective = rcu_dereference_protected(cgrp->bpf.effective[type],
|
|
|
|
|
lockdep_is_held(&cgroup_mutex));
|
|
|
|
|
|
2017-10-03 13:50:22 +08:00
|
|
|
if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE)
|
2019-05-29 05:14:43 +08:00
|
|
|
cnt = bpf_prog_array_length(effective);
|
2017-10-03 13:50:22 +08:00
|
|
|
else
|
|
|
|
|
cnt = prog_list_length(progs);
|
|
|
|
|
|
|
|
|
|
if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (attr->query.prog_cnt == 0 || !prog_ids || !cnt)
|
|
|
|
|
/* return early if user requested only program count + flags */
|
|
|
|
|
return 0;
|
|
|
|
|
if (attr->query.prog_cnt < cnt) {
|
|
|
|
|
cnt = attr->query.prog_cnt;
|
|
|
|
|
ret = -ENOSPC;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
|
2019-05-29 05:14:43 +08:00
|
|
|
return bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
|
2017-10-03 13:50:22 +08:00
|
|
|
} else {
|
|
|
|
|
struct bpf_prog_list *pl;
|
|
|
|
|
u32 id;
|
|
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
|
list_for_each_entry(pl, progs, node) {
|
|
|
|
|
id = pl->prog->aux->id;
|
|
|
|
|
if (copy_to_user(prog_ids + i, &id, sizeof(id)))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
if (++i == cnt)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-19 07:04:24 +08:00
|
|
|
int cgroup_bpf_prog_attach(const union bpf_attr *attr,
|
|
|
|
|
enum bpf_prog_type ptype, struct bpf_prog *prog)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
cgrp = cgroup_get_from_fd(attr->target_fd);
|
|
|
|
|
if (IS_ERR(cgrp))
|
|
|
|
|
return PTR_ERR(cgrp);
|
|
|
|
|
|
|
|
|
|
ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type,
|
|
|
|
|
attr->attach_flags);
|
|
|
|
|
cgroup_put(cgrp);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
|
struct cgroup *cgrp;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
cgrp = cgroup_get_from_fd(attr->target_fd);
|
|
|
|
|
if (IS_ERR(cgrp))
|
|
|
|
|
return PTR_ERR(cgrp);
|
|
|
|
|
|
|
|
|
|
prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
|
|
|
|
|
if (IS_ERR(prog))
|
|
|
|
|
prog = NULL;
|
|
|
|
|
|
|
|
|
|
ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0);
|
|
|
|
|
if (prog)
|
|
|
|
|
bpf_prog_put(prog);
|
|
|
|
|
|
|
|
|
|
cgroup_put(cgrp);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int cgroup_bpf_prog_query(const union bpf_attr *attr,
|
|
|
|
|
union bpf_attr __user *uattr)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
cgrp = cgroup_get_from_fd(attr->query.target_fd);
|
|
|
|
|
if (IS_ERR(cgrp))
|
|
|
|
|
return PTR_ERR(cgrp);
|
|
|
|
|
|
|
|
|
|
ret = cgroup_bpf_query(cgrp, attr, uattr);
|
|
|
|
|
|
|
|
|
|
cgroup_put(cgrp);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2016-11-23 23:52:26 +08:00
|
|
|
/**
|
2016-12-02 00:48:03 +08:00
|
|
|
* __cgroup_bpf_run_filter_skb() - Run a program for packet filtering
|
2017-04-12 02:08:08 +08:00
|
|
|
* @sk: The socket sending or receiving traffic
|
2016-11-23 23:52:26 +08:00
|
|
|
* @skb: The skb that is being sent or received
|
|
|
|
|
* @type: The type of program to be exectuted
|
|
|
|
|
*
|
|
|
|
|
* If no socket is passed, or the socket is not of type INET or INET6,
|
|
|
|
|
* this function does nothing and returns 0.
|
|
|
|
|
*
|
|
|
|
|
* The program type passed in via @type must be suitable for network
|
|
|
|
|
* filtering. No further check is performed to assert that.
|
|
|
|
|
*
|
2019-05-29 07:59:37 +08:00
|
|
|
* For egress packets, this function can return:
|
|
|
|
|
* NET_XMIT_SUCCESS (0) - continue with packet output
|
|
|
|
|
* NET_XMIT_DROP (1) - drop packet and notify TCP to call cwr
|
|
|
|
|
* NET_XMIT_CN (2) - continue with packet output and notify TCP
|
|
|
|
|
* to call cwr
|
|
|
|
|
* -EPERM - drop packet
|
|
|
|
|
*
|
|
|
|
|
* For ingress packets, this function will return -EPERM if any
|
|
|
|
|
* attached program was found and if it returned != 1 during execution.
|
|
|
|
|
* Otherwise 0 is returned.
|
2016-11-23 23:52:26 +08:00
|
|
|
*/
|
2016-12-02 00:48:03 +08:00
|
|
|
int __cgroup_bpf_run_filter_skb(struct sock *sk,
|
|
|
|
|
struct sk_buff *skb,
|
|
|
|
|
enum bpf_attach_type type)
|
2016-11-23 23:52:26 +08:00
|
|
|
{
|
2017-10-03 13:50:21 +08:00
|
|
|
unsigned int offset = skb->data - skb_network_header(skb);
|
|
|
|
|
struct sock *save_sk;
|
2018-10-20 00:57:57 +08:00
|
|
|
void *saved_data_end;
|
2016-11-23 23:52:26 +08:00
|
|
|
struct cgroup *cgrp;
|
2017-10-03 13:50:21 +08:00
|
|
|
int ret;
|
2016-11-23 23:52:26 +08:00
|
|
|
|
|
|
|
|
if (!sk || !sk_fullsock(sk))
|
|
|
|
|
return 0;
|
|
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
|
2016-11-23 23:52:26 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
2017-10-03 13:50:21 +08:00
|
|
|
save_sk = skb->sk;
|
|
|
|
|
skb->sk = sk;
|
|
|
|
|
__skb_push(skb, offset);
|
2018-10-20 00:57:57 +08:00
|
|
|
|
|
|
|
|
/* compute pointers for the bpf prog */
|
|
|
|
|
bpf_compute_and_save_data_end(skb, &saved_data_end);
|
|
|
|
|
|
2019-05-29 07:59:37 +08:00
|
|
|
if (type == BPF_CGROUP_INET_EGRESS) {
|
|
|
|
|
ret = BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(
|
|
|
|
|
cgrp->bpf.effective[type], skb, __bpf_prog_run_save_cb);
|
|
|
|
|
} else {
|
|
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
|
|
|
|
|
__bpf_prog_run_save_cb);
|
|
|
|
|
ret = (ret == 1 ? 0 : -EPERM);
|
|
|
|
|
}
|
2018-10-20 00:57:57 +08:00
|
|
|
bpf_restore_data_end(skb, saved_data_end);
|
2017-10-03 13:50:21 +08:00
|
|
|
__skb_pull(skb, offset);
|
|
|
|
|
skb->sk = save_sk;
|
2019-05-29 07:59:37 +08:00
|
|
|
|
|
|
|
|
return ret;
|
2016-11-23 23:52:26 +08:00
|
|
|
}
|
2016-12-02 00:48:03 +08:00
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb);
|
2016-12-02 00:48:04 +08:00
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* __cgroup_bpf_run_filter_sk() - Run a program on a sock
|
|
|
|
|
* @sk: sock structure to manipulate
|
|
|
|
|
* @type: The type of program to be exectuted
|
|
|
|
|
*
|
|
|
|
|
* socket is passed is expected to be of type INET or INET6.
|
|
|
|
|
*
|
|
|
|
|
* The program type passed in via @type must be suitable for sock
|
|
|
|
|
* filtering. No further check is performed to assert that.
|
|
|
|
|
*
|
|
|
|
|
* This function will return %-EPERM if any if an attached program was found
|
|
|
|
|
* and if it returned != 1 during execution. In all other cases, 0 is returned.
|
|
|
|
|
*/
|
|
|
|
|
int __cgroup_bpf_run_filter_sk(struct sock *sk,
|
|
|
|
|
enum bpf_attach_type type)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
2017-10-03 13:50:21 +08:00
|
|
|
int ret;
|
2016-12-02 00:48:04 +08:00
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN);
|
|
|
|
|
return ret == 1 ? 0 : -EPERM;
|
2016-12-02 00:48:04 +08:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
|
bpf: BPF support for sock_ops
Created a new BPF program type, BPF_PROG_TYPE_SOCK_OPS, and a corresponding
struct that allows BPF programs of this type to access some of the
socket's fields (such as IP addresses, ports, etc.). It uses the
existing bpf cgroups infrastructure so the programs can be attached per
cgroup with full inheritance support. The program will be called at
appropriate times to set relevant connections parameters such as buffer
sizes, SYN and SYN-ACK RTOs, etc., based on connection information such
as IP addresses, port numbers, etc.
Alghough there are already 3 mechanisms to set parameters (sysctls,
route metrics and setsockopts), this new mechanism provides some
distinct advantages. Unlike sysctls, it can set parameters per
connection. In contrast to route metrics, it can also use port numbers
and information provided by a user level program. In addition, it could
set parameters probabilistically for evaluation purposes (i.e. do
something different on 10% of the flows and compare results with the
other 90% of the flows). Also, in cases where IPv6 addresses contain
geographic information, the rules to make changes based on the distance
(or RTT) between the hosts are much easier than route metric rules and
can be global. Finally, unlike setsockopt, it oes not require
application changes and it can be updated easily at any time.
Although the bpf cgroup framework already contains a sock related
program type (BPF_PROG_TYPE_CGROUP_SOCK), I created the new type
(BPF_PROG_TYPE_SOCK_OPS) beccause the existing type expects to be called
only once during the connections's lifetime. In contrast, the new
program type will be called multiple times from different places in the
network stack code. For example, before sending SYN and SYN-ACKs to set
an appropriate timeout, when the connection is established to set
congestion control, etc. As a result it has "op" field to specify the
type of operation requested.
The purpose of this new program type is to simplify setting connection
parameters, such as buffer sizes, TCP's SYN RTO, etc. For example, it is
easy to use facebook's internal IPv6 addresses to determine if both hosts
of a connection are in the same datacenter. Therefore, it is easy to
write a BPF program to choose a small SYN RTO value when both hosts are
in the same datacenter.
This patch only contains the framework to support the new BPF program
type, following patches add the functionality to set various connection
parameters.
This patch defines a new BPF program type: BPF_PROG_TYPE_SOCKET_OPS
and a new bpf syscall command to load a new program of this type:
BPF_PROG_LOAD_SOCKET_OPS.
Two new corresponding structs (one for the kernel one for the user/BPF
program):
/* kernel version */
struct bpf_sock_ops_kern {
struct sock *sk;
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
};
/* user version
* Some fields are in network byte order reflecting the sock struct
* Use the bpf_ntohl helper macro in samples/bpf/bpf_endian.h to
* convert them to host byte order.
*/
struct bpf_sock_ops {
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
__u32 family;
__u32 remote_ip4; /* In network byte order */
__u32 local_ip4; /* In network byte order */
__u32 remote_ip6[4]; /* In network byte order */
__u32 local_ip6[4]; /* In network byte order */
__u32 remote_port; /* In network byte order */
__u32 local_port; /* In host byte horder */
};
Currently there are two types of ops. The first type expects the BPF
program to return a value which is then used by the caller (or a
negative value to indicate the operation is not supported). The second
type expects state changes to be done by the BPF program, for example
through a setsockopt BPF helper function, and they ignore the return
value.
The reply fields of the bpf_sockt_ops struct are there in case a bpf
program needs to return a value larger than an integer.
Signed-off-by: Lawrence Brakmo <brakmo@fb.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-01 11:02:40 +08:00
|
|
|
|
2018-03-31 06:08:02 +08:00
|
|
|
/**
|
|
|
|
|
* __cgroup_bpf_run_filter_sock_addr() - Run a program on a sock and
|
|
|
|
|
* provided by user sockaddr
|
|
|
|
|
* @sk: sock struct that will use sockaddr
|
|
|
|
|
* @uaddr: sockaddr struct provided by user
|
|
|
|
|
* @type: The type of program to be exectuted
|
2018-05-25 23:55:23 +08:00
|
|
|
* @t_ctx: Pointer to attach type specific context
|
2018-03-31 06:08:02 +08:00
|
|
|
*
|
|
|
|
|
* socket is expected to be of type INET or INET6.
|
|
|
|
|
*
|
|
|
|
|
* This function will return %-EPERM if an attached program is found and
|
|
|
|
|
* returned value != 1 during execution. In all other cases, 0 is returned.
|
|
|
|
|
*/
|
|
|
|
|
int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
|
|
|
|
|
struct sockaddr *uaddr,
|
2018-05-25 23:55:23 +08:00
|
|
|
enum bpf_attach_type type,
|
|
|
|
|
void *t_ctx)
|
2018-03-31 06:08:02 +08:00
|
|
|
{
|
|
|
|
|
struct bpf_sock_addr_kern ctx = {
|
|
|
|
|
.sk = sk,
|
|
|
|
|
.uaddr = uaddr,
|
2018-05-25 23:55:23 +08:00
|
|
|
.t_ctx = t_ctx,
|
2018-03-31 06:08:02 +08:00
|
|
|
};
|
2018-05-25 23:55:23 +08:00
|
|
|
struct sockaddr_storage unspec;
|
2018-03-31 06:08:02 +08:00
|
|
|
struct cgroup *cgrp;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
/* Check socket family since not all sockets represent network
|
|
|
|
|
* endpoint (e.g. AF_UNIX).
|
|
|
|
|
*/
|
|
|
|
|
if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
|
|
|
|
|
return 0;
|
|
|
|
|
|
2018-05-25 23:55:23 +08:00
|
|
|
if (!ctx.uaddr) {
|
|
|
|
|
memset(&unspec, 0, sizeof(unspec));
|
|
|
|
|
ctx.uaddr = (struct sockaddr *)&unspec;
|
|
|
|
|
}
|
|
|
|
|
|
2018-03-31 06:08:02 +08:00
|
|
|
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
|
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
|
|
|
|
|
|
|
|
|
|
return ret == 1 ? 0 : -EPERM;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);
|
|
|
|
|
|
bpf: BPF support for sock_ops
Created a new BPF program type, BPF_PROG_TYPE_SOCK_OPS, and a corresponding
struct that allows BPF programs of this type to access some of the
socket's fields (such as IP addresses, ports, etc.). It uses the
existing bpf cgroups infrastructure so the programs can be attached per
cgroup with full inheritance support. The program will be called at
appropriate times to set relevant connections parameters such as buffer
sizes, SYN and SYN-ACK RTOs, etc., based on connection information such
as IP addresses, port numbers, etc.
Alghough there are already 3 mechanisms to set parameters (sysctls,
route metrics and setsockopts), this new mechanism provides some
distinct advantages. Unlike sysctls, it can set parameters per
connection. In contrast to route metrics, it can also use port numbers
and information provided by a user level program. In addition, it could
set parameters probabilistically for evaluation purposes (i.e. do
something different on 10% of the flows and compare results with the
other 90% of the flows). Also, in cases where IPv6 addresses contain
geographic information, the rules to make changes based on the distance
(or RTT) between the hosts are much easier than route metric rules and
can be global. Finally, unlike setsockopt, it oes not require
application changes and it can be updated easily at any time.
Although the bpf cgroup framework already contains a sock related
program type (BPF_PROG_TYPE_CGROUP_SOCK), I created the new type
(BPF_PROG_TYPE_SOCK_OPS) beccause the existing type expects to be called
only once during the connections's lifetime. In contrast, the new
program type will be called multiple times from different places in the
network stack code. For example, before sending SYN and SYN-ACKs to set
an appropriate timeout, when the connection is established to set
congestion control, etc. As a result it has "op" field to specify the
type of operation requested.
The purpose of this new program type is to simplify setting connection
parameters, such as buffer sizes, TCP's SYN RTO, etc. For example, it is
easy to use facebook's internal IPv6 addresses to determine if both hosts
of a connection are in the same datacenter. Therefore, it is easy to
write a BPF program to choose a small SYN RTO value when both hosts are
in the same datacenter.
This patch only contains the framework to support the new BPF program
type, following patches add the functionality to set various connection
parameters.
This patch defines a new BPF program type: BPF_PROG_TYPE_SOCKET_OPS
and a new bpf syscall command to load a new program of this type:
BPF_PROG_LOAD_SOCKET_OPS.
Two new corresponding structs (one for the kernel one for the user/BPF
program):
/* kernel version */
struct bpf_sock_ops_kern {
struct sock *sk;
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
};
/* user version
* Some fields are in network byte order reflecting the sock struct
* Use the bpf_ntohl helper macro in samples/bpf/bpf_endian.h to
* convert them to host byte order.
*/
struct bpf_sock_ops {
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
__u32 family;
__u32 remote_ip4; /* In network byte order */
__u32 local_ip4; /* In network byte order */
__u32 remote_ip6[4]; /* In network byte order */
__u32 local_ip6[4]; /* In network byte order */
__u32 remote_port; /* In network byte order */
__u32 local_port; /* In host byte horder */
};
Currently there are two types of ops. The first type expects the BPF
program to return a value which is then used by the caller (or a
negative value to indicate the operation is not supported). The second
type expects state changes to be done by the BPF program, for example
through a setsockopt BPF helper function, and they ignore the return
value.
The reply fields of the bpf_sockt_ops struct are there in case a bpf
program needs to return a value larger than an integer.
Signed-off-by: Lawrence Brakmo <brakmo@fb.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-01 11:02:40 +08:00
|
|
|
/**
|
|
|
|
|
* __cgroup_bpf_run_filter_sock_ops() - Run a program on a sock
|
|
|
|
|
* @sk: socket to get cgroup from
|
|
|
|
|
* @sock_ops: bpf_sock_ops_kern struct to pass to program. Contains
|
|
|
|
|
* sk with connection information (IP addresses, etc.) May not contain
|
|
|
|
|
* cgroup info if it is a req sock.
|
|
|
|
|
* @type: The type of program to be exectuted
|
|
|
|
|
*
|
|
|
|
|
* socket passed is expected to be of type INET or INET6.
|
|
|
|
|
*
|
|
|
|
|
* The program type passed in via @type must be suitable for sock_ops
|
|
|
|
|
* filtering. No further check is performed to assert that.
|
|
|
|
|
*
|
|
|
|
|
* This function will return %-EPERM if any if an attached program was found
|
|
|
|
|
* and if it returned != 1 during execution. In all other cases, 0 is returned.
|
|
|
|
|
*/
|
|
|
|
|
int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
|
|
|
|
|
struct bpf_sock_ops_kern *sock_ops,
|
|
|
|
|
enum bpf_attach_type type)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
2017-10-03 13:50:21 +08:00
|
|
|
int ret;
|
bpf: BPF support for sock_ops
Created a new BPF program type, BPF_PROG_TYPE_SOCK_OPS, and a corresponding
struct that allows BPF programs of this type to access some of the
socket's fields (such as IP addresses, ports, etc.). It uses the
existing bpf cgroups infrastructure so the programs can be attached per
cgroup with full inheritance support. The program will be called at
appropriate times to set relevant connections parameters such as buffer
sizes, SYN and SYN-ACK RTOs, etc., based on connection information such
as IP addresses, port numbers, etc.
Alghough there are already 3 mechanisms to set parameters (sysctls,
route metrics and setsockopts), this new mechanism provides some
distinct advantages. Unlike sysctls, it can set parameters per
connection. In contrast to route metrics, it can also use port numbers
and information provided by a user level program. In addition, it could
set parameters probabilistically for evaluation purposes (i.e. do
something different on 10% of the flows and compare results with the
other 90% of the flows). Also, in cases where IPv6 addresses contain
geographic information, the rules to make changes based on the distance
(or RTT) between the hosts are much easier than route metric rules and
can be global. Finally, unlike setsockopt, it oes not require
application changes and it can be updated easily at any time.
Although the bpf cgroup framework already contains a sock related
program type (BPF_PROG_TYPE_CGROUP_SOCK), I created the new type
(BPF_PROG_TYPE_SOCK_OPS) beccause the existing type expects to be called
only once during the connections's lifetime. In contrast, the new
program type will be called multiple times from different places in the
network stack code. For example, before sending SYN and SYN-ACKs to set
an appropriate timeout, when the connection is established to set
congestion control, etc. As a result it has "op" field to specify the
type of operation requested.
The purpose of this new program type is to simplify setting connection
parameters, such as buffer sizes, TCP's SYN RTO, etc. For example, it is
easy to use facebook's internal IPv6 addresses to determine if both hosts
of a connection are in the same datacenter. Therefore, it is easy to
write a BPF program to choose a small SYN RTO value when both hosts are
in the same datacenter.
This patch only contains the framework to support the new BPF program
type, following patches add the functionality to set various connection
parameters.
This patch defines a new BPF program type: BPF_PROG_TYPE_SOCKET_OPS
and a new bpf syscall command to load a new program of this type:
BPF_PROG_LOAD_SOCKET_OPS.
Two new corresponding structs (one for the kernel one for the user/BPF
program):
/* kernel version */
struct bpf_sock_ops_kern {
struct sock *sk;
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
};
/* user version
* Some fields are in network byte order reflecting the sock struct
* Use the bpf_ntohl helper macro in samples/bpf/bpf_endian.h to
* convert them to host byte order.
*/
struct bpf_sock_ops {
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
__u32 family;
__u32 remote_ip4; /* In network byte order */
__u32 local_ip4; /* In network byte order */
__u32 remote_ip6[4]; /* In network byte order */
__u32 local_ip6[4]; /* In network byte order */
__u32 remote_port; /* In network byte order */
__u32 local_port; /* In host byte horder */
};
Currently there are two types of ops. The first type expects the BPF
program to return a value which is then used by the caller (or a
negative value to indicate the operation is not supported). The second
type expects state changes to be done by the BPF program, for example
through a setsockopt BPF helper function, and they ignore the return
value.
The reply fields of the bpf_sockt_ops struct are there in case a bpf
program needs to return a value larger than an integer.
Signed-off-by: Lawrence Brakmo <brakmo@fb.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-01 11:02:40 +08:00
|
|
|
|
2017-10-03 13:50:21 +08:00
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops,
|
|
|
|
|
BPF_PROG_RUN);
|
|
|
|
|
return ret == 1 ? 0 : -EPERM;
|
bpf: BPF support for sock_ops
Created a new BPF program type, BPF_PROG_TYPE_SOCK_OPS, and a corresponding
struct that allows BPF programs of this type to access some of the
socket's fields (such as IP addresses, ports, etc.). It uses the
existing bpf cgroups infrastructure so the programs can be attached per
cgroup with full inheritance support. The program will be called at
appropriate times to set relevant connections parameters such as buffer
sizes, SYN and SYN-ACK RTOs, etc., based on connection information such
as IP addresses, port numbers, etc.
Alghough there are already 3 mechanisms to set parameters (sysctls,
route metrics and setsockopts), this new mechanism provides some
distinct advantages. Unlike sysctls, it can set parameters per
connection. In contrast to route metrics, it can also use port numbers
and information provided by a user level program. In addition, it could
set parameters probabilistically for evaluation purposes (i.e. do
something different on 10% of the flows and compare results with the
other 90% of the flows). Also, in cases where IPv6 addresses contain
geographic information, the rules to make changes based on the distance
(or RTT) between the hosts are much easier than route metric rules and
can be global. Finally, unlike setsockopt, it oes not require
application changes and it can be updated easily at any time.
Although the bpf cgroup framework already contains a sock related
program type (BPF_PROG_TYPE_CGROUP_SOCK), I created the new type
(BPF_PROG_TYPE_SOCK_OPS) beccause the existing type expects to be called
only once during the connections's lifetime. In contrast, the new
program type will be called multiple times from different places in the
network stack code. For example, before sending SYN and SYN-ACKs to set
an appropriate timeout, when the connection is established to set
congestion control, etc. As a result it has "op" field to specify the
type of operation requested.
The purpose of this new program type is to simplify setting connection
parameters, such as buffer sizes, TCP's SYN RTO, etc. For example, it is
easy to use facebook's internal IPv6 addresses to determine if both hosts
of a connection are in the same datacenter. Therefore, it is easy to
write a BPF program to choose a small SYN RTO value when both hosts are
in the same datacenter.
This patch only contains the framework to support the new BPF program
type, following patches add the functionality to set various connection
parameters.
This patch defines a new BPF program type: BPF_PROG_TYPE_SOCKET_OPS
and a new bpf syscall command to load a new program of this type:
BPF_PROG_LOAD_SOCKET_OPS.
Two new corresponding structs (one for the kernel one for the user/BPF
program):
/* kernel version */
struct bpf_sock_ops_kern {
struct sock *sk;
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
};
/* user version
* Some fields are in network byte order reflecting the sock struct
* Use the bpf_ntohl helper macro in samples/bpf/bpf_endian.h to
* convert them to host byte order.
*/
struct bpf_sock_ops {
__u32 op;
union {
__u32 reply;
__u32 replylong[4];
};
__u32 family;
__u32 remote_ip4; /* In network byte order */
__u32 local_ip4; /* In network byte order */
__u32 remote_ip6[4]; /* In network byte order */
__u32 local_ip6[4]; /* In network byte order */
__u32 remote_port; /* In network byte order */
__u32 local_port; /* In host byte horder */
};
Currently there are two types of ops. The first type expects the BPF
program to return a value which is then used by the caller (or a
negative value to indicate the operation is not supported). The second
type expects state changes to be done by the BPF program, for example
through a setsockopt BPF helper function, and they ignore the return
value.
The reply fields of the bpf_sockt_ops struct are there in case a bpf
program needs to return a value larger than an integer.
Signed-off-by: Lawrence Brakmo <brakmo@fb.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-01 11:02:40 +08:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
|
2017-11-05 21:15:32 +08:00
|
|
|
|
|
|
|
|
int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
|
|
|
|
|
short access, enum bpf_attach_type type)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp;
|
|
|
|
|
struct bpf_cgroup_dev_ctx ctx = {
|
|
|
|
|
.access_type = (access << 16) | dev_type,
|
|
|
|
|
.major = major,
|
|
|
|
|
.minor = minor,
|
|
|
|
|
};
|
|
|
|
|
int allow = 1;
|
|
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
cgrp = task_dfl_cgroup(current);
|
|
|
|
|
allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx,
|
|
|
|
|
BPF_PROG_RUN);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
|
|
return !allow;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_check_dev_permission);
|
|
|
|
|
|
|
|
|
|
static const struct bpf_func_proto *
|
2019-03-13 00:27:09 +08:00
|
|
|
cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
2017-11-05 21:15:32 +08:00
|
|
|
{
|
|
|
|
|
switch (func_id) {
|
|
|
|
|
case BPF_FUNC_map_lookup_elem:
|
|
|
|
|
return &bpf_map_lookup_elem_proto;
|
|
|
|
|
case BPF_FUNC_map_update_elem:
|
|
|
|
|
return &bpf_map_update_elem_proto;
|
|
|
|
|
case BPF_FUNC_map_delete_elem:
|
|
|
|
|
return &bpf_map_delete_elem_proto;
|
2019-04-15 00:58:46 +08:00
|
|
|
case BPF_FUNC_map_push_elem:
|
|
|
|
|
return &bpf_map_push_elem_proto;
|
|
|
|
|
case BPF_FUNC_map_pop_elem:
|
|
|
|
|
return &bpf_map_pop_elem_proto;
|
|
|
|
|
case BPF_FUNC_map_peek_elem:
|
|
|
|
|
return &bpf_map_peek_elem_proto;
|
2017-11-05 21:15:32 +08:00
|
|
|
case BPF_FUNC_get_current_uid_gid:
|
|
|
|
|
return &bpf_get_current_uid_gid_proto;
|
2018-08-03 05:27:24 +08:00
|
|
|
case BPF_FUNC_get_local_storage:
|
|
|
|
|
return &bpf_get_local_storage_proto;
|
2018-09-28 05:37:30 +08:00
|
|
|
case BPF_FUNC_get_current_cgroup_id:
|
|
|
|
|
return &bpf_get_current_cgroup_id_proto;
|
2017-11-05 21:15:32 +08:00
|
|
|
case BPF_FUNC_trace_printk:
|
|
|
|
|
if (capable(CAP_SYS_ADMIN))
|
|
|
|
|
return bpf_get_trace_printk_proto();
|
2019-01-17 03:35:29 +08:00
|
|
|
/* fall through */
|
2017-11-05 21:15:32 +08:00
|
|
|
default:
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-13 00:27:09 +08:00
|
|
|
static const struct bpf_func_proto *
|
|
|
|
|
cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|
|
|
|
{
|
|
|
|
|
return cgroup_base_func_proto(func_id, prog);
|
|
|
|
|
}
|
|
|
|
|
|
2017-11-05 21:15:32 +08:00
|
|
|
static bool cgroup_dev_is_valid_access(int off, int size,
|
|
|
|
|
enum bpf_access_type type,
|
2018-03-31 06:08:00 +08:00
|
|
|
const struct bpf_prog *prog,
|
2017-11-05 21:15:32 +08:00
|
|
|
struct bpf_insn_access_aux *info)
|
|
|
|
|
{
|
2017-12-19 02:13:44 +08:00
|
|
|
const int size_default = sizeof(__u32);
|
|
|
|
|
|
2017-11-05 21:15:32 +08:00
|
|
|
if (type == BPF_WRITE)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx))
|
|
|
|
|
return false;
|
|
|
|
|
/* The verifier guarantees that size > 0. */
|
|
|
|
|
if (off % size != 0)
|
|
|
|
|
return false;
|
2017-12-19 02:13:44 +08:00
|
|
|
|
|
|
|
|
switch (off) {
|
|
|
|
|
case bpf_ctx_range(struct bpf_cgroup_dev_ctx, access_type):
|
|
|
|
|
bpf_ctx_record_field_size(info, size_default);
|
|
|
|
|
if (!bpf_ctx_narrow_access_ok(off, size, size_default))
|
|
|
|
|
return false;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
if (size != size_default)
|
|
|
|
|
return false;
|
|
|
|
|
}
|
2017-11-05 21:15:32 +08:00
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const struct bpf_prog_ops cg_dev_prog_ops = {
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const struct bpf_verifier_ops cg_dev_verifier_ops = {
|
|
|
|
|
.get_func_proto = cgroup_dev_func_proto,
|
|
|
|
|
.is_valid_access = cgroup_dev_is_valid_access,
|
|
|
|
|
};
|
2019-02-28 04:59:24 +08:00
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* __cgroup_bpf_run_filter_sysctl - Run a program on sysctl
|
|
|
|
|
*
|
|
|
|
|
* @head: sysctl table header
|
|
|
|
|
* @table: sysctl table
|
|
|
|
|
* @write: sysctl is being read (= 0) or written (= 1)
|
2019-03-08 10:38:43 +08:00
|
|
|
* @buf: pointer to buffer passed by user space
|
|
|
|
|
* @pcount: value-result argument: value is size of buffer pointed to by @buf,
|
|
|
|
|
* result is size of @new_buf if program set new value, initial value
|
|
|
|
|
* otherwise
|
2019-03-08 10:50:52 +08:00
|
|
|
* @ppos: value-result argument: value is position at which read from or write
|
|
|
|
|
* to sysctl is happening, result is new position if program overrode it,
|
|
|
|
|
* initial value otherwise
|
2019-03-08 10:38:43 +08:00
|
|
|
* @new_buf: pointer to pointer to new buffer that will be allocated if program
|
|
|
|
|
* overrides new value provided by user space on sysctl write
|
|
|
|
|
* NOTE: it's caller responsibility to free *new_buf if it was set
|
2019-02-28 04:59:24 +08:00
|
|
|
* @type: type of program to be executed
|
|
|
|
|
*
|
|
|
|
|
* Program is run when sysctl is being accessed, either read or written, and
|
|
|
|
|
* can allow or deny such access.
|
|
|
|
|
*
|
|
|
|
|
* This function will return %-EPERM if an attached program is found and
|
|
|
|
|
* returned value != 1 during execution. In all other cases 0 is returned.
|
|
|
|
|
*/
|
|
|
|
|
int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
|
|
|
|
|
struct ctl_table *table, int write,
|
2019-03-08 10:38:43 +08:00
|
|
|
void __user *buf, size_t *pcount,
|
2019-03-08 10:50:52 +08:00
|
|
|
loff_t *ppos, void **new_buf,
|
|
|
|
|
enum bpf_attach_type type)
|
2019-02-28 04:59:24 +08:00
|
|
|
{
|
|
|
|
|
struct bpf_sysctl_kern ctx = {
|
|
|
|
|
.head = head,
|
|
|
|
|
.table = table,
|
|
|
|
|
.write = write,
|
2019-03-08 10:50:52 +08:00
|
|
|
.ppos = ppos,
|
bpf: Introduce bpf_sysctl_get_current_value helper
Add bpf_sysctl_get_current_value() helper to copy current sysctl value
into provided by BPF_PROG_TYPE_CGROUP_SYSCTL program buffer.
It provides same string as user space can see by reading corresponding
file in /proc/sys/, including new line, etc.
Documentation for the new helper is provided in bpf.h UAPI.
Since current value is kept in ctl_table->data in a parsed form,
ctl_table->proc_handler() with write=0 is called to read that data and
convert it to a string. Such a string can later be parsed by a program
using helpers that will be introduced separately.
Unfortunately it's not trivial to provide API to access parsed data due to
variety of data representations (string, intvec, uintvec, ulongvec,
custom structures, even NULL, etc). Instead it's assumed that user know
how to handle specific sysctl they're interested in and appropriate
helpers can be used.
Since ctl_table->proc_handler() expects __user buffer, conversion to
__user happens for kernel allocated one where the value is stored.
Signed-off-by: Andrey Ignatov <rdna@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-01 11:22:15 +08:00
|
|
|
.cur_val = NULL,
|
|
|
|
|
.cur_len = PAGE_SIZE,
|
2019-03-08 10:38:43 +08:00
|
|
|
.new_val = NULL,
|
|
|
|
|
.new_len = 0,
|
|
|
|
|
.new_updated = 0,
|
2019-02-28 04:59:24 +08:00
|
|
|
};
|
|
|
|
|
struct cgroup *cgrp;
|
|
|
|
|
int ret;
|
|
|
|
|
|
bpf: Introduce bpf_sysctl_get_current_value helper
Add bpf_sysctl_get_current_value() helper to copy current sysctl value
into provided by BPF_PROG_TYPE_CGROUP_SYSCTL program buffer.
It provides same string as user space can see by reading corresponding
file in /proc/sys/, including new line, etc.
Documentation for the new helper is provided in bpf.h UAPI.
Since current value is kept in ctl_table->data in a parsed form,
ctl_table->proc_handler() with write=0 is called to read that data and
convert it to a string. Such a string can later be parsed by a program
using helpers that will be introduced separately.
Unfortunately it's not trivial to provide API to access parsed data due to
variety of data representations (string, intvec, uintvec, ulongvec,
custom structures, even NULL, etc). Instead it's assumed that user know
how to handle specific sysctl they're interested in and appropriate
helpers can be used.
Since ctl_table->proc_handler() expects __user buffer, conversion to
__user happens for kernel allocated one where the value is stored.
Signed-off-by: Andrey Ignatov <rdna@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-01 11:22:15 +08:00
|
|
|
ctx.cur_val = kmalloc_track_caller(ctx.cur_len, GFP_KERNEL);
|
|
|
|
|
if (ctx.cur_val) {
|
|
|
|
|
mm_segment_t old_fs;
|
|
|
|
|
loff_t pos = 0;
|
|
|
|
|
|
|
|
|
|
old_fs = get_fs();
|
|
|
|
|
set_fs(KERNEL_DS);
|
|
|
|
|
if (table->proc_handler(table, 0, (void __user *)ctx.cur_val,
|
|
|
|
|
&ctx.cur_len, &pos)) {
|
|
|
|
|
/* Let BPF program decide how to proceed. */
|
|
|
|
|
ctx.cur_len = 0;
|
|
|
|
|
}
|
|
|
|
|
set_fs(old_fs);
|
|
|
|
|
} else {
|
|
|
|
|
/* Let BPF program decide how to proceed. */
|
|
|
|
|
ctx.cur_len = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-08 10:38:43 +08:00
|
|
|
if (write && buf && *pcount) {
|
|
|
|
|
/* BPF program should be able to override new value with a
|
|
|
|
|
* buffer bigger than provided by user.
|
|
|
|
|
*/
|
|
|
|
|
ctx.new_val = kmalloc_track_caller(PAGE_SIZE, GFP_KERNEL);
|
2019-04-13 07:01:01 +08:00
|
|
|
ctx.new_len = min_t(size_t, PAGE_SIZE, *pcount);
|
2019-03-08 10:38:43 +08:00
|
|
|
if (!ctx.new_val ||
|
|
|
|
|
copy_from_user(ctx.new_val, buf, ctx.new_len))
|
|
|
|
|
/* Let BPF program decide how to proceed. */
|
|
|
|
|
ctx.new_len = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-28 04:59:24 +08:00
|
|
|
rcu_read_lock();
|
|
|
|
|
cgrp = task_dfl_cgroup(current);
|
|
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
bpf: Introduce bpf_sysctl_get_current_value helper
Add bpf_sysctl_get_current_value() helper to copy current sysctl value
into provided by BPF_PROG_TYPE_CGROUP_SYSCTL program buffer.
It provides same string as user space can see by reading corresponding
file in /proc/sys/, including new line, etc.
Documentation for the new helper is provided in bpf.h UAPI.
Since current value is kept in ctl_table->data in a parsed form,
ctl_table->proc_handler() with write=0 is called to read that data and
convert it to a string. Such a string can later be parsed by a program
using helpers that will be introduced separately.
Unfortunately it's not trivial to provide API to access parsed data due to
variety of data representations (string, intvec, uintvec, ulongvec,
custom structures, even NULL, etc). Instead it's assumed that user know
how to handle specific sysctl they're interested in and appropriate
helpers can be used.
Since ctl_table->proc_handler() expects __user buffer, conversion to
__user happens for kernel allocated one where the value is stored.
Signed-off-by: Andrey Ignatov <rdna@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-01 11:22:15 +08:00
|
|
|
kfree(ctx.cur_val);
|
|
|
|
|
|
2019-03-08 10:38:43 +08:00
|
|
|
if (ret == 1 && ctx.new_updated) {
|
|
|
|
|
*new_buf = ctx.new_val;
|
|
|
|
|
*pcount = ctx.new_len;
|
|
|
|
|
} else {
|
|
|
|
|
kfree(ctx.new_val);
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-28 04:59:24 +08:00
|
|
|
return ret == 1 ? 0 : -EPERM;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sysctl);
|
|
|
|
|
|
bpf: implement getsockopt and setsockopt hooks
Implement new BPF_PROG_TYPE_CGROUP_SOCKOPT program type and
BPF_CGROUP_{G,S}ETSOCKOPT cgroup hooks.
BPF_CGROUP_SETSOCKOPT can modify user setsockopt arguments before
passing them down to the kernel or bypass kernel completely.
BPF_CGROUP_GETSOCKOPT can can inspect/modify getsockopt arguments that
kernel returns.
Both hooks reuse existing PTR_TO_PACKET{,_END} infrastructure.
The buffer memory is pre-allocated (because I don't think there is
a precedent for working with __user memory from bpf). This might be
slow to do for each {s,g}etsockopt call, that's why I've added
__cgroup_bpf_prog_array_is_empty that exits early if there is nothing
attached to a cgroup. Note, however, that there is a race between
__cgroup_bpf_prog_array_is_empty and BPF_PROG_RUN_ARRAY where cgroup
program layout might have changed; this should not be a problem
because in general there is a race between multiple calls to
{s,g}etsocktop and user adding/removing bpf progs from a cgroup.
The return code of the BPF program is handled as follows:
* 0: EPERM
* 1: success, continue with next BPF program in the cgroup chain
v9:
* allow overwriting setsockopt arguments (Alexei Starovoitov):
* use set_fs (same as kernel_setsockopt)
* buffer is always kzalloc'd (no small on-stack buffer)
v8:
* use s32 for optlen (Andrii Nakryiko)
v7:
* return only 0 or 1 (Alexei Starovoitov)
* always run all progs (Alexei Starovoitov)
* use optval=0 as kernel bypass in setsockopt (Alexei Starovoitov)
(decided to use optval=-1 instead, optval=0 might be a valid input)
* call getsockopt hook after kernel handlers (Alexei Starovoitov)
v6:
* rework cgroup chaining; stop as soon as bpf program returns
0 or 2; see patch with the documentation for the details
* drop Andrii's and Martin's Acked-by (not sure they are comfortable
with the new state of things)
v5:
* skip copy_to_user() and put_user() when ret == 0 (Martin Lau)
v4:
* don't export bpf_sk_fullsock helper (Martin Lau)
* size != sizeof(__u64) for uapi pointers (Martin Lau)
* offsetof instead of bpf_ctx_range when checking ctx access (Martin Lau)
v3:
* typos in BPF_PROG_CGROUP_SOCKOPT_RUN_ARRAY comments (Andrii Nakryiko)
* reverse christmas tree in BPF_PROG_CGROUP_SOCKOPT_RUN_ARRAY (Andrii
Nakryiko)
* use __bpf_md_ptr instead of __u32 for optval{,_end} (Martin Lau)
* use BPF_FIELD_SIZEOF() for consistency (Martin Lau)
* new CG_SOCKOPT_ACCESS macro to wrap repeated parts
v2:
* moved bpf_sockopt_kern fields around to remove a hole (Martin Lau)
* aligned bpf_sockopt_kern->buf to 8 bytes (Martin Lau)
* bpf_prog_array_is_empty instead of bpf_prog_array_length (Martin Lau)
* added [0,2] return code check to verifier (Martin Lau)
* dropped unused buf[64] from the stack (Martin Lau)
* use PTR_TO_SOCKET for bpf_sockopt->sk (Martin Lau)
* dropped bpf_target_off from ctx rewrites (Martin Lau)
* use return code for kernel bypass (Martin Lau & Andrii Nakryiko)
Cc: Andrii Nakryiko <andriin@fb.com>
Cc: Martin Lau <kafai@fb.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-06-28 04:38:47 +08:00
|
|
|
static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,
|
|
|
|
|
enum bpf_attach_type attach_type)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_prog_array *prog_array;
|
|
|
|
|
bool empty;
|
|
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]);
|
|
|
|
|
empty = bpf_prog_array_is_empty(prog_array);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
|
|
return empty;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
|
|
|
|
|
{
|
|
|
|
|
if (unlikely(max_optlen > PAGE_SIZE) || max_optlen < 0)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
ctx->optval = kzalloc(max_optlen, GFP_USER);
|
|
|
|
|
if (!ctx->optval)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
ctx->optval_end = ctx->optval + max_optlen;
|
|
|
|
|
ctx->optlen = max_optlen;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
|
|
|
|
|
{
|
|
|
|
|
kfree(ctx->optval);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
|
|
|
|
|
int *optname, char __user *optval,
|
|
|
|
|
int *optlen, char **kernel_optval)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
|
|
|
|
struct bpf_sockopt_kern ctx = {
|
|
|
|
|
.sk = sk,
|
|
|
|
|
.level = *level,
|
|
|
|
|
.optname = *optname,
|
|
|
|
|
};
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
/* Opportunistic check to see whether we have any BPF program
|
|
|
|
|
* attached to the hook so we don't waste time allocating
|
|
|
|
|
* memory and locking the socket.
|
|
|
|
|
*/
|
|
|
|
|
if (!cgroup_bpf_enabled ||
|
|
|
|
|
__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
ret = sockopt_alloc_buf(&ctx, *optlen);
|
|
|
|
|
if (ret)
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(ctx.optval, optval, *optlen) != 0) {
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
lock_sock(sk);
|
|
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT],
|
|
|
|
|
&ctx, BPF_PROG_RUN);
|
|
|
|
|
release_sock(sk);
|
|
|
|
|
|
|
|
|
|
if (!ret) {
|
|
|
|
|
ret = -EPERM;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ctx.optlen == -1) {
|
|
|
|
|
/* optlen set to -1, bypass kernel */
|
|
|
|
|
ret = 1;
|
|
|
|
|
} else if (ctx.optlen > *optlen || ctx.optlen < -1) {
|
|
|
|
|
/* optlen is out of bounds */
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
} else {
|
|
|
|
|
/* optlen within bounds, run kernel handler */
|
|
|
|
|
ret = 0;
|
|
|
|
|
|
|
|
|
|
/* export any potential modifications */
|
|
|
|
|
*level = ctx.level;
|
|
|
|
|
*optname = ctx.optname;
|
|
|
|
|
*optlen = ctx.optlen;
|
|
|
|
|
*kernel_optval = ctx.optval;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
if (ret)
|
|
|
|
|
sockopt_free_buf(&ctx);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_setsockopt);
|
|
|
|
|
|
|
|
|
|
int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
|
|
|
|
|
int optname, char __user *optval,
|
|
|
|
|
int __user *optlen, int max_optlen,
|
|
|
|
|
int retval)
|
|
|
|
|
{
|
|
|
|
|
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
|
|
|
|
struct bpf_sockopt_kern ctx = {
|
|
|
|
|
.sk = sk,
|
|
|
|
|
.level = level,
|
|
|
|
|
.optname = optname,
|
|
|
|
|
.retval = retval,
|
|
|
|
|
};
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
/* Opportunistic check to see whether we have any BPF program
|
|
|
|
|
* attached to the hook so we don't waste time allocating
|
|
|
|
|
* memory and locking the socket.
|
|
|
|
|
*/
|
|
|
|
|
if (!cgroup_bpf_enabled ||
|
|
|
|
|
__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
|
|
|
|
|
return retval;
|
|
|
|
|
|
|
|
|
|
ret = sockopt_alloc_buf(&ctx, max_optlen);
|
|
|
|
|
if (ret)
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
|
|
if (!retval) {
|
|
|
|
|
/* If kernel getsockopt finished successfully,
|
|
|
|
|
* copy whatever was returned to the user back
|
|
|
|
|
* into our temporary buffer. Set optlen to the
|
|
|
|
|
* one that kernel returned as well to let
|
|
|
|
|
* BPF programs inspect the value.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
if (get_user(ctx.optlen, optlen)) {
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ctx.optlen > max_optlen)
|
|
|
|
|
ctx.optlen = max_optlen;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(ctx.optval, optval, ctx.optlen) != 0) {
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
lock_sock(sk);
|
|
|
|
|
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
|
|
|
|
|
&ctx, BPF_PROG_RUN);
|
|
|
|
|
release_sock(sk);
|
|
|
|
|
|
|
|
|
|
if (!ret) {
|
|
|
|
|
ret = -EPERM;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ctx.optlen > max_optlen) {
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* BPF programs only allowed to set retval to 0, not some
|
|
|
|
|
* arbitrary value.
|
|
|
|
|
*/
|
|
|
|
|
if (ctx.retval != 0 && ctx.retval != retval) {
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (copy_to_user(optval, ctx.optval, ctx.optlen) ||
|
|
|
|
|
put_user(ctx.optlen, optlen)) {
|
|
|
|
|
ret = -EFAULT;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ret = ctx.retval;
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
sockopt_free_buf(&ctx);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(__cgroup_bpf_run_filter_getsockopt);
|
|
|
|
|
|
2019-02-28 05:28:48 +08:00
|
|
|
static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
|
|
|
|
|
size_t *lenp)
|
|
|
|
|
{
|
|
|
|
|
ssize_t tmp_ret = 0, ret;
|
|
|
|
|
|
|
|
|
|
if (dir->header.parent) {
|
|
|
|
|
tmp_ret = sysctl_cpy_dir(dir->header.parent, bufp, lenp);
|
|
|
|
|
if (tmp_ret < 0)
|
|
|
|
|
return tmp_ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ret = strscpy(*bufp, dir->header.ctl_table[0].procname, *lenp);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
return ret;
|
|
|
|
|
*bufp += ret;
|
|
|
|
|
*lenp -= ret;
|
|
|
|
|
ret += tmp_ret;
|
|
|
|
|
|
|
|
|
|
/* Avoid leading slash. */
|
|
|
|
|
if (!ret)
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
|
|
tmp_ret = strscpy(*bufp, "/", *lenp);
|
|
|
|
|
if (tmp_ret < 0)
|
|
|
|
|
return tmp_ret;
|
|
|
|
|
*bufp += tmp_ret;
|
|
|
|
|
*lenp -= tmp_ret;
|
|
|
|
|
|
|
|
|
|
return ret + tmp_ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BPF_CALL_4(bpf_sysctl_get_name, struct bpf_sysctl_kern *, ctx, char *, buf,
|
|
|
|
|
size_t, buf_len, u64, flags)
|
|
|
|
|
{
|
|
|
|
|
ssize_t tmp_ret = 0, ret;
|
|
|
|
|
|
|
|
|
|
if (!buf)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
if (!(flags & BPF_F_SYSCTL_BASE_NAME)) {
|
|
|
|
|
if (!ctx->head)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
tmp_ret = sysctl_cpy_dir(ctx->head->parent, &buf, &buf_len);
|
|
|
|
|
if (tmp_ret < 0)
|
|
|
|
|
return tmp_ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ret = strscpy(buf, ctx->table->procname, buf_len);
|
|
|
|
|
|
|
|
|
|
return ret < 0 ? ret : tmp_ret + ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
|
|
|
|
|
.func = bpf_sysctl_get_name,
|
|
|
|
|
.gpl_only = false,
|
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
|
.arg1_type = ARG_PTR_TO_CTX,
|
|
|
|
|
.arg2_type = ARG_PTR_TO_MEM,
|
|
|
|
|
.arg3_type = ARG_CONST_SIZE,
|
|
|
|
|
.arg4_type = ARG_ANYTHING,
|
|
|
|
|
};
|
|
|
|
|
|
bpf: Introduce bpf_sysctl_get_current_value helper
Add bpf_sysctl_get_current_value() helper to copy current sysctl value
into provided by BPF_PROG_TYPE_CGROUP_SYSCTL program buffer.
It provides same string as user space can see by reading corresponding
file in /proc/sys/, including new line, etc.
Documentation for the new helper is provided in bpf.h UAPI.
Since current value is kept in ctl_table->data in a parsed form,
ctl_table->proc_handler() with write=0 is called to read that data and
convert it to a string. Such a string can later be parsed by a program
using helpers that will be introduced separately.
Unfortunately it's not trivial to provide API to access parsed data due to
variety of data representations (string, intvec, uintvec, ulongvec,
custom structures, even NULL, etc). Instead it's assumed that user know
how to handle specific sysctl they're interested in and appropriate
helpers can be used.
Since ctl_table->proc_handler() expects __user buffer, conversion to
__user happens for kernel allocated one where the value is stored.
Signed-off-by: Andrey Ignatov <rdna@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-01 11:22:15 +08:00
|
|
|
static int copy_sysctl_value(char *dst, size_t dst_len, char *src,
|
|
|
|
|
size_t src_len)
|
|
|
|
|
{
|
|
|
|
|
if (!dst)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
if (!dst_len)
|
|
|
|
|
return -E2BIG;
|
|
|
|
|
|
|
|
|
|
if (!src || !src_len) {
|
|
|
|
|
memset(dst, 0, dst_len);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
memcpy(dst, src, min(dst_len, src_len));
|
|
|
|
|
|
|
|
|
|
if (dst_len > src_len) {
|
|
|
|
|
memset(dst + src_len, '\0', dst_len - src_len);
|
|
|
|
|
return src_len;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dst[dst_len - 1] = '\0';
|
|
|
|
|
|
|
|
|
|
return -E2BIG;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BPF_CALL_3(bpf_sysctl_get_current_value, struct bpf_sysctl_kern *, ctx,
|
|
|
|
|
char *, buf, size_t, buf_len)
|
|
|
|
|
{
|
|
|
|
|
return copy_sysctl_value(buf, buf_len, ctx->cur_val, ctx->cur_len);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = {
|
|
|
|
|
.func = bpf_sysctl_get_current_value,
|
|
|
|
|
.gpl_only = false,
|
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
|
.arg1_type = ARG_PTR_TO_CTX,
|
|
|
|
|
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
|
|
|
|
.arg3_type = ARG_CONST_SIZE,
|
|
|
|
|
};
|
|
|
|
|
|
2019-03-08 10:38:43 +08:00
|
|
|
BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf,
|
|
|
|
|
size_t, buf_len)
|
|
|
|
|
{
|
|
|
|
|
if (!ctx->write) {
|
|
|
|
|
if (buf && buf_len)
|
|
|
|
|
memset(buf, '\0', buf_len);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
return copy_sysctl_value(buf, buf_len, ctx->new_val, ctx->new_len);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = {
|
|
|
|
|
.func = bpf_sysctl_get_new_value,
|
|
|
|
|
.gpl_only = false,
|
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
|
.arg1_type = ARG_PTR_TO_CTX,
|
|
|
|
|
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
|
|
|
|
.arg3_type = ARG_CONST_SIZE,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx,
|
|
|
|
|
const char *, buf, size_t, buf_len)
|
|
|
|
|
{
|
|
|
|
|
if (!ctx->write || !ctx->new_val || !ctx->new_len || !buf || !buf_len)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
if (buf_len > PAGE_SIZE - 1)
|
|
|
|
|
return -E2BIG;
|
|
|
|
|
|
|
|
|
|
memcpy(ctx->new_val, buf, buf_len);
|
|
|
|
|
ctx->new_len = buf_len;
|
|
|
|
|
ctx->new_updated = 1;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
|
|
|
|
|
.func = bpf_sysctl_set_new_value,
|
|
|
|
|
.gpl_only = false,
|
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
|
.arg1_type = ARG_PTR_TO_CTX,
|
|
|
|
|
.arg2_type = ARG_PTR_TO_MEM,
|
|
|
|
|
.arg3_type = ARG_CONST_SIZE,
|
|
|
|
|
};
|
|
|
|
|
|
2019-02-28 04:59:24 +08:00
|
|
|
static const struct bpf_func_proto *
|
|
|
|
|
sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|
|
|
|
{
|
2019-02-28 05:28:48 +08:00
|
|
|
switch (func_id) {
|
2019-03-19 08:55:26 +08:00
|
|
|
case BPF_FUNC_strtol:
|
|
|
|
|
return &bpf_strtol_proto;
|
|
|
|
|
case BPF_FUNC_strtoul:
|
|
|
|
|
return &bpf_strtoul_proto;
|
2019-02-28 05:28:48 +08:00
|
|
|
case BPF_FUNC_sysctl_get_name:
|
|
|
|
|
return &bpf_sysctl_get_name_proto;
|
bpf: Introduce bpf_sysctl_get_current_value helper
Add bpf_sysctl_get_current_value() helper to copy current sysctl value
into provided by BPF_PROG_TYPE_CGROUP_SYSCTL program buffer.
It provides same string as user space can see by reading corresponding
file in /proc/sys/, including new line, etc.
Documentation for the new helper is provided in bpf.h UAPI.
Since current value is kept in ctl_table->data in a parsed form,
ctl_table->proc_handler() with write=0 is called to read that data and
convert it to a string. Such a string can later be parsed by a program
using helpers that will be introduced separately.
Unfortunately it's not trivial to provide API to access parsed data due to
variety of data representations (string, intvec, uintvec, ulongvec,
custom structures, even NULL, etc). Instead it's assumed that user know
how to handle specific sysctl they're interested in and appropriate
helpers can be used.
Since ctl_table->proc_handler() expects __user buffer, conversion to
__user happens for kernel allocated one where the value is stored.
Signed-off-by: Andrey Ignatov <rdna@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-01 11:22:15 +08:00
|
|
|
case BPF_FUNC_sysctl_get_current_value:
|
|
|
|
|
return &bpf_sysctl_get_current_value_proto;
|
2019-03-08 10:38:43 +08:00
|
|
|
case BPF_FUNC_sysctl_get_new_value:
|
|
|
|
|
return &bpf_sysctl_get_new_value_proto;
|
|
|
|
|
case BPF_FUNC_sysctl_set_new_value:
|
|
|
|
|
return &bpf_sysctl_set_new_value_proto;
|
2019-02-28 05:28:48 +08:00
|
|
|
default:
|
|
|
|
|
return cgroup_base_func_proto(func_id, prog);
|
|
|
|
|
}
|
2019-02-28 04:59:24 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type,
|
|
|
|
|
const struct bpf_prog *prog,
|
|
|
|
|
struct bpf_insn_access_aux *info)
|
|
|
|
|
{
|
|
|
|
|
const int size_default = sizeof(__u32);
|
|
|
|
|
|
2019-03-08 10:50:52 +08:00
|
|
|
if (off < 0 || off + size > sizeof(struct bpf_sysctl) || off % size)
|
2019-02-28 04:59:24 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
switch (off) {
|
|
|
|
|
case offsetof(struct bpf_sysctl, write):
|
2019-03-08 10:50:52 +08:00
|
|
|
if (type != BPF_READ)
|
|
|
|
|
return false;
|
2019-02-28 04:59:24 +08:00
|
|
|
bpf_ctx_record_field_size(info, size_default);
|
|
|
|
|
return bpf_ctx_narrow_access_ok(off, size, size_default);
|
2019-03-08 10:50:52 +08:00
|
|
|
case offsetof(struct bpf_sysctl, file_pos):
|
|
|
|
|
if (type == BPF_READ) {
|
|
|
|
|
bpf_ctx_record_field_size(info, size_default);
|
|
|
|
|
return bpf_ctx_narrow_access_ok(off, size, size_default);
|
|
|
|
|
} else {
|
|
|
|
|
return size == size_default;
|
|
|
|
|
}
|
2019-02-28 04:59:24 +08:00
|
|
|
default:
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static u32 sysctl_convert_ctx_access(enum bpf_access_type type,
|
|
|
|
|
const struct bpf_insn *si,
|
|
|
|
|
struct bpf_insn *insn_buf,
|
|
|
|
|
struct bpf_prog *prog, u32 *target_size)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_insn *insn = insn_buf;
|
|
|
|
|
|
|
|
|
|
switch (si->off) {
|
|
|
|
|
case offsetof(struct bpf_sysctl, write):
|
|
|
|
|
*insn++ = BPF_LDX_MEM(
|
|
|
|
|
BPF_SIZE(si->code), si->dst_reg, si->src_reg,
|
|
|
|
|
bpf_target_off(struct bpf_sysctl_kern, write,
|
|
|
|
|
FIELD_SIZEOF(struct bpf_sysctl_kern,
|
|
|
|
|
write),
|
|
|
|
|
target_size));
|
|
|
|
|
break;
|
2019-03-08 10:50:52 +08:00
|
|
|
case offsetof(struct bpf_sysctl, file_pos):
|
|
|
|
|
/* ppos is a pointer so it should be accessed via indirect
|
|
|
|
|
* loads and stores. Also for stores additional temporary
|
|
|
|
|
* register is used since neither src_reg nor dst_reg can be
|
|
|
|
|
* overridden.
|
|
|
|
|
*/
|
|
|
|
|
if (type == BPF_WRITE) {
|
|
|
|
|
int treg = BPF_REG_9;
|
|
|
|
|
|
|
|
|
|
if (si->src_reg == treg || si->dst_reg == treg)
|
|
|
|
|
--treg;
|
|
|
|
|
if (si->src_reg == treg || si->dst_reg == treg)
|
|
|
|
|
--treg;
|
|
|
|
|
*insn++ = BPF_STX_MEM(
|
|
|
|
|
BPF_DW, si->dst_reg, treg,
|
|
|
|
|
offsetof(struct bpf_sysctl_kern, tmp_reg));
|
|
|
|
|
*insn++ = BPF_LDX_MEM(
|
|
|
|
|
BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
|
|
|
|
|
treg, si->dst_reg,
|
|
|
|
|
offsetof(struct bpf_sysctl_kern, ppos));
|
|
|
|
|
*insn++ = BPF_STX_MEM(
|
|
|
|
|
BPF_SIZEOF(u32), treg, si->src_reg, 0);
|
|
|
|
|
*insn++ = BPF_LDX_MEM(
|
|
|
|
|
BPF_DW, treg, si->dst_reg,
|
|
|
|
|
offsetof(struct bpf_sysctl_kern, tmp_reg));
|
|
|
|
|
} else {
|
|
|
|
|
*insn++ = BPF_LDX_MEM(
|
|
|
|
|
BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
|
|
|
|
|
si->dst_reg, si->src_reg,
|
|
|
|
|
offsetof(struct bpf_sysctl_kern, ppos));
|
|
|
|
|
*insn++ = BPF_LDX_MEM(
|
|
|
|
|
BPF_SIZE(si->code), si->dst_reg, si->dst_reg, 0);
|
|
|
|
|
}
|
|
|
|
|
*target_size = sizeof(u32);
|
|
|
|
|
break;
|
2019-02-28 04:59:24 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return insn - insn_buf;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const struct bpf_verifier_ops cg_sysctl_verifier_ops = {
|
|
|
|
|
.get_func_proto = sysctl_func_proto,
|
|
|
|
|
.is_valid_access = sysctl_is_valid_access,
|
|
|
|
|
.convert_ctx_access = sysctl_convert_ctx_access,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const struct bpf_prog_ops cg_sysctl_prog_ops = {
|
|
|
|
|
};
|
bpf: implement getsockopt and setsockopt hooks
Implement new BPF_PROG_TYPE_CGROUP_SOCKOPT program type and
BPF_CGROUP_{G,S}ETSOCKOPT cgroup hooks.
BPF_CGROUP_SETSOCKOPT can modify user setsockopt arguments before
passing them down to the kernel or bypass kernel completely.
BPF_CGROUP_GETSOCKOPT can can inspect/modify getsockopt arguments that
kernel returns.
Both hooks reuse existing PTR_TO_PACKET{,_END} infrastructure.
The buffer memory is pre-allocated (because I don't think there is
a precedent for working with __user memory from bpf). This might be
slow to do for each {s,g}etsockopt call, that's why I've added
__cgroup_bpf_prog_array_is_empty that exits early if there is nothing
attached to a cgroup. Note, however, that there is a race between
__cgroup_bpf_prog_array_is_empty and BPF_PROG_RUN_ARRAY where cgroup
program layout might have changed; this should not be a problem
because in general there is a race between multiple calls to
{s,g}etsocktop and user adding/removing bpf progs from a cgroup.
The return code of the BPF program is handled as follows:
* 0: EPERM
* 1: success, continue with next BPF program in the cgroup chain
v9:
* allow overwriting setsockopt arguments (Alexei Starovoitov):
* use set_fs (same as kernel_setsockopt)
* buffer is always kzalloc'd (no small on-stack buffer)
v8:
* use s32 for optlen (Andrii Nakryiko)
v7:
* return only 0 or 1 (Alexei Starovoitov)
* always run all progs (Alexei Starovoitov)
* use optval=0 as kernel bypass in setsockopt (Alexei Starovoitov)
(decided to use optval=-1 instead, optval=0 might be a valid input)
* call getsockopt hook after kernel handlers (Alexei Starovoitov)
v6:
* rework cgroup chaining; stop as soon as bpf program returns
0 or 2; see patch with the documentation for the details
* drop Andrii's and Martin's Acked-by (not sure they are comfortable
with the new state of things)
v5:
* skip copy_to_user() and put_user() when ret == 0 (Martin Lau)
v4:
* don't export bpf_sk_fullsock helper (Martin Lau)
* size != sizeof(__u64) for uapi pointers (Martin Lau)
* offsetof instead of bpf_ctx_range when checking ctx access (Martin Lau)
v3:
* typos in BPF_PROG_CGROUP_SOCKOPT_RUN_ARRAY comments (Andrii Nakryiko)
* reverse christmas tree in BPF_PROG_CGROUP_SOCKOPT_RUN_ARRAY (Andrii
Nakryiko)
* use __bpf_md_ptr instead of __u32 for optval{,_end} (Martin Lau)
* use BPF_FIELD_SIZEOF() for consistency (Martin Lau)
* new CG_SOCKOPT_ACCESS macro to wrap repeated parts
v2:
* moved bpf_sockopt_kern fields around to remove a hole (Martin Lau)
* aligned bpf_sockopt_kern->buf to 8 bytes (Martin Lau)
* bpf_prog_array_is_empty instead of bpf_prog_array_length (Martin Lau)
* added [0,2] return code check to verifier (Martin Lau)
* dropped unused buf[64] from the stack (Martin Lau)
* use PTR_TO_SOCKET for bpf_sockopt->sk (Martin Lau)
* dropped bpf_target_off from ctx rewrites (Martin Lau)
* use return code for kernel bypass (Martin Lau & Andrii Nakryiko)
Cc: Andrii Nakryiko <andriin@fb.com>
Cc: Martin Lau <kafai@fb.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-06-28 04:38:47 +08:00
|
|
|
|
|
|
|
|
static const struct bpf_func_proto *
|
|
|
|
|
cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|
|
|
|
{
|
|
|
|
|
switch (func_id) {
|
|
|
|
|
case BPF_FUNC_sk_storage_get:
|
|
|
|
|
return &bpf_sk_storage_get_proto;
|
|
|
|
|
case BPF_FUNC_sk_storage_delete:
|
|
|
|
|
return &bpf_sk_storage_delete_proto;
|
|
|
|
|
#ifdef CONFIG_INET
|
|
|
|
|
case BPF_FUNC_tcp_sock:
|
|
|
|
|
return &bpf_tcp_sock_proto;
|
|
|
|
|
#endif
|
|
|
|
|
default:
|
|
|
|
|
return cgroup_base_func_proto(func_id, prog);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static bool cg_sockopt_is_valid_access(int off, int size,
|
|
|
|
|
enum bpf_access_type type,
|
|
|
|
|
const struct bpf_prog *prog,
|
|
|
|
|
struct bpf_insn_access_aux *info)
|
|
|
|
|
{
|
|
|
|
|
const int size_default = sizeof(__u32);
|
|
|
|
|
|
|
|
|
|
if (off < 0 || off >= sizeof(struct bpf_sockopt))
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
if (off % size != 0)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
if (type == BPF_WRITE) {
|
|
|
|
|
switch (off) {
|
|
|
|
|
case offsetof(struct bpf_sockopt, retval):
|
|
|
|
|
if (size != size_default)
|
|
|
|
|
return false;
|
|
|
|
|
return prog->expected_attach_type ==
|
|
|
|
|
BPF_CGROUP_GETSOCKOPT;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optname):
|
|
|
|
|
/* fallthrough */
|
|
|
|
|
case offsetof(struct bpf_sockopt, level):
|
|
|
|
|
if (size != size_default)
|
|
|
|
|
return false;
|
|
|
|
|
return prog->expected_attach_type ==
|
|
|
|
|
BPF_CGROUP_SETSOCKOPT;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optlen):
|
|
|
|
|
return size == size_default;
|
|
|
|
|
default:
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
switch (off) {
|
|
|
|
|
case offsetof(struct bpf_sockopt, sk):
|
|
|
|
|
if (size != sizeof(__u64))
|
|
|
|
|
return false;
|
|
|
|
|
info->reg_type = PTR_TO_SOCKET;
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optval):
|
|
|
|
|
if (size != sizeof(__u64))
|
|
|
|
|
return false;
|
|
|
|
|
info->reg_type = PTR_TO_PACKET;
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optval_end):
|
|
|
|
|
if (size != sizeof(__u64))
|
|
|
|
|
return false;
|
|
|
|
|
info->reg_type = PTR_TO_PACKET_END;
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, retval):
|
|
|
|
|
if (size != size_default)
|
|
|
|
|
return false;
|
|
|
|
|
return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;
|
|
|
|
|
default:
|
|
|
|
|
if (size != size_default)
|
|
|
|
|
return false;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define CG_SOCKOPT_ACCESS_FIELD(T, F) \
|
|
|
|
|
T(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F), \
|
|
|
|
|
si->dst_reg, si->src_reg, \
|
|
|
|
|
offsetof(struct bpf_sockopt_kern, F))
|
|
|
|
|
|
|
|
|
|
static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type,
|
|
|
|
|
const struct bpf_insn *si,
|
|
|
|
|
struct bpf_insn *insn_buf,
|
|
|
|
|
struct bpf_prog *prog,
|
|
|
|
|
u32 *target_size)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_insn *insn = insn_buf;
|
|
|
|
|
|
|
|
|
|
switch (si->off) {
|
|
|
|
|
case offsetof(struct bpf_sockopt, sk):
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, sk);
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, level):
|
|
|
|
|
if (type == BPF_WRITE)
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, level);
|
|
|
|
|
else
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, level);
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optname):
|
|
|
|
|
if (type == BPF_WRITE)
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optname);
|
|
|
|
|
else
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optname);
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optlen):
|
|
|
|
|
if (type == BPF_WRITE)
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optlen);
|
|
|
|
|
else
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen);
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, retval):
|
|
|
|
|
if (type == BPF_WRITE)
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval);
|
|
|
|
|
else
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval);
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optval):
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval);
|
|
|
|
|
break;
|
|
|
|
|
case offsetof(struct bpf_sockopt, optval_end):
|
|
|
|
|
*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval_end);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return insn - insn_buf;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf,
|
|
|
|
|
bool direct_write,
|
|
|
|
|
const struct bpf_prog *prog)
|
|
|
|
|
{
|
|
|
|
|
/* Nothing to do for sockopt argument. The data is kzalloc'ated.
|
|
|
|
|
*/
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
|
|
|
|
|
.get_func_proto = cg_sockopt_func_proto,
|
|
|
|
|
.is_valid_access = cg_sockopt_is_valid_access,
|
|
|
|
|
.convert_ctx_access = cg_sockopt_convert_ctx_access,
|
|
|
|
|
.gen_prologue = cg_sockopt_get_prologue,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const struct bpf_prog_ops cg_sockopt_prog_ops = {
|
|
|
|
|
};
|