hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

epoll: add EPOLLEXCLUSIVE flag 

Currently, epoll file descriptors or epfds (the fd returned from
epoll_create[1]()) that are added to a shared wakeup source are always
added in a non-exclusive manner.  This means that when we have multiple
epfds attached to a shared fd source they are all woken up.  This creates
thundering herd type behavior.

Introduce a new 'EPOLLEXCLUSIVE' flag that can be passed as part of the
'event' argument during an epoll_ctl() EPOLL_CTL_ADD operation.  This new
flag allows for exclusive wakeups when there are multiple epfds attached
to a shared fd event source.

The implementation walks the list of exclusive waiters, and queues an
event to each epfd, until it finds the first waiter that has threads
blocked on it via epoll_wait().  The idea is to search for threads which
are idle and ready to process the wakeup events.  Thus, we queue an event
to at least 1 epfd, but may still potentially queue an event to all epfds
that are attached to the shared fd source.

Performance testing was done by Madars Vitolins using a modified version
of Enduro/X.  The use of the 'EPOLLEXCLUSIVE' flag reduce the length of
this particular workload from 860s down to 24s.

Sample epoll_clt text:

EPOLLEXCLUSIVE

  Sets an exclusive wakeup mode for the epfd file descriptor that is
  being attached to the target file descriptor, fd.  Thus, when an event
  occurs and multiple epfd file descriptors are attached to the same
  target file using EPOLLEXCLUSIVE, one or more epfds will receive an
  event with epoll_wait(2).  The default in this scenario (when
  EPOLLEXCLUSIVE is not set) is for all epfds to receive an event.
  EPOLLEXCLUSIVE may only be specified with the op EPOLL_CTL_ADD.

Signed-off-by: Jason Baron <jbaron@akamai.com>
Tested-by: Madars Vitolins <m@silodev.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Al Viro <viro@ftp.linux.org.uk>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Eric Wong <normalperson@yhbt.net>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Hagen Paul Pfeifer <hagen@jauu.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Jason Baron 2016-01-20 14:59:24 -08:00 committed by Linus Torvalds
parent 6b10df4257
commit df0108c5da
2 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
fs/eventpoll.c
View File

@ -92,7 +92,7 @@
*/ */
/* Epoll private bits inside the event mask */ /* Epoll private bits inside the event mask */
#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET) #define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)
/* Maximum number of nesting allowed inside epoll sets */ /* Maximum number of nesting allowed inside epoll sets */
#define EP_MAX_NESTS 4 #define EP_MAX_NESTS 4
@ -1002,6 +1002,7 @@ static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *k
unsigned long flags; unsigned long flags;
struct epitem *epi = ep_item_from_wait(wait); struct epitem *epi = ep_item_from_wait(wait);
struct eventpoll *ep = epi->ep; struct eventpoll *ep = epi->ep;
int ewake = 0;
if ((unsigned long)key & POLLFREE) { if ((unsigned long)key & POLLFREE) {
ep_pwq_from_wait(wait)->whead = NULL; ep_pwq_from_wait(wait)->whead = NULL;
@ -1066,8 +1067,10 @@ static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *k
* Wake up ( if active ) both the eventpoll wait list and the ->poll() * Wake up ( if active ) both the eventpoll wait list and the ->poll()
* wait list. * wait list.
*/ */
if (waitqueue_active(&ep->wq)) if (waitqueue_active(&ep->wq)) {
ewake = 1;
wake_up_locked(&ep->wq); wake_up_locked(&ep->wq);
}
if (waitqueue_active(&ep->poll_wait)) if (waitqueue_active(&ep->poll_wait))
pwake++; pwake++;
@ -1078,6 +1081,9 @@ out_unlock:
if (pwake) if (pwake)
ep_poll_safewake(&ep->poll_wait); ep_poll_safewake(&ep->poll_wait);
if (epi->event.events & EPOLLEXCLUSIVE)
return ewake;
return 1; return 1;
} }
@ -1095,7 +1101,10 @@ static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
init_waitqueue_func_entry(&pwq->wait, ep_poll_callback); init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
pwq->whead = whead; pwq->whead = whead;
pwq->base = epi; pwq->base = epi;
add_wait_queue(whead, &pwq->wait); if (epi->event.events & EPOLLEXCLUSIVE)
add_wait_queue_exclusive(whead, &pwq->wait);
else
add_wait_queue(whead, &pwq->wait);
list_add_tail(&pwq->llink, &epi->pwqlist); list_add_tail(&pwq->llink, &epi->pwqlist);
epi->nwait++; epi->nwait++;
} else { } else {
@ -1861,6 +1870,15 @@ SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
if (f.file == tf.file || !is_file_epoll(f.file)) if (f.file == tf.file || !is_file_epoll(f.file))
goto error_tgt_fput; goto error_tgt_fput;
/*
* epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,
* so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.
* Also, we do not currently supported nested exclusive wakeups.
*/
if ((epds.events & EPOLLEXCLUSIVE) && (op == EPOLL_CTL_MOD ||
(op == EPOLL_CTL_ADD && is_file_epoll(tf.file))))
goto error_tgt_fput;
/* /*
* At this point it is safe to assume that the "private_data" contains * At this point it is safe to assume that the "private_data" contains
* our own data structure. * our own data structure.

3
include/uapi/linux/eventpoll.h
View File

@ -26,6 +26,9 @@
#define EPOLL_CTL_DEL 2 #define EPOLL_CTL_DEL 2
#define EPOLL_CTL_MOD 3 #define EPOLL_CTL_MOD 3
/* Set exclusive wakeup mode for the target file descriptor */
#define EPOLLEXCLUSIVE (1 << 28)
/* /*
* Request the handling of system wakeup events so as to prevent system suspends * Request the handling of system wakeup events so as to prevent system suspends
* from happening while those events are being processed. * from happening while those events are being processed.