2011-04-15 06:25:47 +08:00
|
|
|
/*
|
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU General Public License version 2
|
|
|
|
|
* as published by the Free Software Foundation; or, when distributed
|
|
|
|
|
* separately from the Linux kernel or incorporated into other
|
|
|
|
|
* software packages, subject to the following license:
|
|
|
|
|
*
|
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
|
|
* of this source file (the "Software"), to deal in the Software without
|
|
|
|
|
* restriction, including without limitation the rights to use, copy, modify,
|
|
|
|
|
* merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
|
|
|
|
* and to permit persons to whom the Software is furnished to do so, subject to
|
|
|
|
|
* the following conditions:
|
|
|
|
|
*
|
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
|
*
|
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
|
*/
|
|
|
|
|
|
2011-05-13 04:58:21 +08:00
|
|
|
#ifndef __XEN_BLKIF__BACKEND__COMMON_H__
|
|
|
|
|
#define __XEN_BLKIF__BACKEND__COMMON_H__
|
2011-04-15 06:25:47 +08:00
|
|
|
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
|
#include <linux/slab.h>
|
|
|
|
|
#include <linux/blkdev.h>
|
|
|
|
|
#include <linux/vmalloc.h>
|
|
|
|
|
#include <linux/wait.h>
|
2011-04-15 05:21:50 +08:00
|
|
|
#include <linux/io.h>
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
#include <linux/rbtree.h>
|
2011-04-15 06:25:47 +08:00
|
|
|
#include <asm/setup.h>
|
|
|
|
|
#include <asm/pgalloc.h>
|
|
|
|
|
#include <asm/hypervisor.h>
|
2009-02-10 04:05:51 +08:00
|
|
|
#include <xen/grant_table.h>
|
2015-05-05 23:25:56 +08:00
|
|
|
#include <xen/page.h>
|
2011-04-15 06:25:47 +08:00
|
|
|
#include <xen/xenbus.h>
|
2011-05-13 04:31:51 +08:00
|
|
|
#include <xen/interface/io/ring.h>
|
|
|
|
|
#include <xen/interface/io/blkif.h>
|
|
|
|
|
#include <xen/interface/io/protocols.h>
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2015-06-03 13:40:03 +08:00
|
|
|
extern unsigned int xen_blkif_max_ring_order;
|
2015-11-14 11:12:17 +08:00
|
|
|
extern unsigned int xenblk_max_queues;
|
2013-04-18 22:06:54 +08:00
|
|
|
/*
|
|
|
|
|
* This is the maximum number of segments that would be allowed in indirect
|
|
|
|
|
* requests. This value will also be passed to the frontend.
|
|
|
|
|
*/
|
|
|
|
|
#define MAX_INDIRECT_SEGMENTS 256
|
|
|
|
|
|
2015-05-05 23:25:56 +08:00
|
|
|
/*
|
|
|
|
|
* Xen use 4K pages. The guest may use different page size (4K or 64K)
|
|
|
|
|
* Number of Xen pages per segment
|
|
|
|
|
*/
|
|
|
|
|
#define XEN_PAGES_PER_SEGMENT (PAGE_SIZE / XEN_PAGE_SIZE)
|
|
|
|
|
|
|
|
|
|
#define XEN_PAGES_PER_INDIRECT_FRAME \
|
|
|
|
|
(XEN_PAGE_SIZE/sizeof(struct blkif_request_segment))
|
|
|
|
|
#define SEGS_PER_INDIRECT_FRAME \
|
|
|
|
|
(XEN_PAGES_PER_INDIRECT_FRAME / XEN_PAGES_PER_SEGMENT)
|
|
|
|
|
|
2013-04-18 22:06:54 +08:00
|
|
|
#define MAX_INDIRECT_PAGES \
|
|
|
|
|
((MAX_INDIRECT_SEGMENTS + SEGS_PER_INDIRECT_FRAME - 1)/SEGS_PER_INDIRECT_FRAME)
|
2015-05-05 23:25:56 +08:00
|
|
|
#define INDIRECT_PAGES(_segs) DIV_ROUND_UP(_segs, XEN_PAGES_PER_INDIRECT_FRAME)
|
2013-04-18 22:06:54 +08:00
|
|
|
|
2011-05-13 04:31:51 +08:00
|
|
|
/* Not a real protocol. Used to generate ring structs which contain
|
|
|
|
|
* the elements common to all protocols only. This way we get a
|
|
|
|
|
* compiler-checkable way to use common struct elements, so we can
|
|
|
|
|
* avoid using switch(protocol) in a number of places. */
|
|
|
|
|
struct blkif_common_request {
|
|
|
|
|
char dummy;
|
|
|
|
|
};
|
2017-06-14 04:28:27 +08:00
|
|
|
|
|
|
|
|
/* i386 protocol version */
|
2011-05-13 04:31:51 +08:00
|
|
|
|
2011-09-01 18:39:10 +08:00
|
|
|
struct blkif_x86_32_request_rw {
|
2011-10-13 00:12:36 +08:00
|
|
|
uint8_t nr_segments; /* number of segments */
|
|
|
|
|
blkif_vdev_t handle; /* only for read/write requests */
|
|
|
|
|
uint64_t id; /* private guest value, echoed in resp */
|
2011-09-01 18:39:10 +08:00
|
|
|
blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */
|
|
|
|
|
struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
2011-10-13 00:12:36 +08:00
|
|
|
} __attribute__((__packed__));
|
2011-09-01 18:39:10 +08:00
|
|
|
|
|
|
|
|
struct blkif_x86_32_request_discard {
|
2011-10-13 04:23:30 +08:00
|
|
|
uint8_t flag; /* BLKIF_DISCARD_SECURE or zero */
|
2011-10-13 00:12:36 +08:00
|
|
|
blkif_vdev_t _pad1; /* was "handle" for read/write requests */
|
|
|
|
|
uint64_t id; /* private guest value, echoed in resp */
|
2011-09-01 18:39:10 +08:00
|
|
|
blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */
|
2011-10-13 00:12:36 +08:00
|
|
|
uint64_t nr_sectors;
|
|
|
|
|
} __attribute__((__packed__));
|
2011-09-01 18:39:10 +08:00
|
|
|
|
2013-03-08 01:32:01 +08:00
|
|
|
struct blkif_x86_32_request_other {
|
|
|
|
|
uint8_t _pad1;
|
|
|
|
|
blkif_vdev_t _pad2;
|
|
|
|
|
uint64_t id; /* private guest value, echoed in resp */
|
|
|
|
|
} __attribute__((__packed__));
|
|
|
|
|
|
2013-04-18 22:06:54 +08:00
|
|
|
struct blkif_x86_32_request_indirect {
|
|
|
|
|
uint8_t indirect_op;
|
|
|
|
|
uint16_t nr_segments;
|
|
|
|
|
uint64_t id;
|
|
|
|
|
blkif_sector_t sector_number;
|
|
|
|
|
blkif_vdev_t handle;
|
|
|
|
|
uint16_t _pad1;
|
|
|
|
|
grant_ref_t indirect_grefs[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST];
|
|
|
|
|
/*
|
|
|
|
|
* The maximum number of indirect segments (and pages) that will
|
|
|
|
|
* be used is determined by MAX_INDIRECT_SEGMENTS, this value
|
|
|
|
|
* is also exported to the guest (via xenstore
|
|
|
|
|
* feature-max-indirect-segments entry), so the frontend knows how
|
|
|
|
|
* many indirect segments the backend supports.
|
|
|
|
|
*/
|
|
|
|
|
uint64_t _pad2; /* make it 64 byte aligned */
|
|
|
|
|
} __attribute__((__packed__));
|
|
|
|
|
|
2011-05-13 04:31:51 +08:00
|
|
|
struct blkif_x86_32_request {
|
|
|
|
|
uint8_t operation; /* BLKIF_OP_??? */
|
2011-09-01 18:39:10 +08:00
|
|
|
union {
|
|
|
|
|
struct blkif_x86_32_request_rw rw;
|
|
|
|
|
struct blkif_x86_32_request_discard discard;
|
2013-03-08 01:32:01 +08:00
|
|
|
struct blkif_x86_32_request_other other;
|
2013-04-18 22:06:54 +08:00
|
|
|
struct blkif_x86_32_request_indirect indirect;
|
2011-09-01 18:39:10 +08:00
|
|
|
} u;
|
2011-10-13 00:12:36 +08:00
|
|
|
} __attribute__((__packed__));
|
|
|
|
|
|
2011-05-13 04:31:51 +08:00
|
|
|
/* x86_64 protocol version */
|
2011-09-01 18:39:10 +08:00
|
|
|
|
|
|
|
|
struct blkif_x86_64_request_rw {
|
2011-10-13 00:12:36 +08:00
|
|
|
uint8_t nr_segments; /* number of segments */
|
|
|
|
|
blkif_vdev_t handle; /* only for read/write requests */
|
|
|
|
|
uint32_t _pad1; /* offsetof(blkif_reqest..,u.rw.id)==8 */
|
|
|
|
|
uint64_t id;
|
2011-09-01 18:39:10 +08:00
|
|
|
blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */
|
|
|
|
|
struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
2011-10-13 00:12:36 +08:00
|
|
|
} __attribute__((__packed__));
|
2011-09-01 18:39:10 +08:00
|
|
|
|
|
|
|
|
struct blkif_x86_64_request_discard {
|
2011-10-13 04:23:30 +08:00
|
|
|
uint8_t flag; /* BLKIF_DISCARD_SECURE or zero */
|
2011-10-13 00:12:36 +08:00
|
|
|
blkif_vdev_t _pad1; /* was "handle" for read/write requests */
|
|
|
|
|
uint32_t _pad2; /* offsetof(blkif_..,u.discard.id)==8 */
|
|
|
|
|
uint64_t id;
|
2011-09-01 18:39:10 +08:00
|
|
|
blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */
|
2011-10-13 00:12:36 +08:00
|
|
|
uint64_t nr_sectors;
|
|
|
|
|
} __attribute__((__packed__));
|
2011-09-01 18:39:10 +08:00
|
|
|
|
2013-03-08 01:32:01 +08:00
|
|
|
struct blkif_x86_64_request_other {
|
|
|
|
|
uint8_t _pad1;
|
|
|
|
|
blkif_vdev_t _pad2;
|
|
|
|
|
uint32_t _pad3; /* offsetof(blkif_..,u.discard.id)==8 */
|
|
|
|
|
uint64_t id; /* private guest value, echoed in resp */
|
|
|
|
|
} __attribute__((__packed__));
|
|
|
|
|
|
2013-04-18 22:06:54 +08:00
|
|
|
struct blkif_x86_64_request_indirect {
|
|
|
|
|
uint8_t indirect_op;
|
|
|
|
|
uint16_t nr_segments;
|
|
|
|
|
uint32_t _pad1; /* offsetof(blkif_..,u.indirect.id)==8 */
|
|
|
|
|
uint64_t id;
|
|
|
|
|
blkif_sector_t sector_number;
|
|
|
|
|
blkif_vdev_t handle;
|
|
|
|
|
uint16_t _pad2;
|
|
|
|
|
grant_ref_t indirect_grefs[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST];
|
|
|
|
|
/*
|
|
|
|
|
* The maximum number of indirect segments (and pages) that will
|
|
|
|
|
* be used is determined by MAX_INDIRECT_SEGMENTS, this value
|
|
|
|
|
* is also exported to the guest (via xenstore
|
|
|
|
|
* feature-max-indirect-segments entry), so the frontend knows how
|
|
|
|
|
* many indirect segments the backend supports.
|
|
|
|
|
*/
|
|
|
|
|
uint32_t _pad3; /* make it 64 byte aligned */
|
|
|
|
|
} __attribute__((__packed__));
|
|
|
|
|
|
2011-05-13 04:31:51 +08:00
|
|
|
struct blkif_x86_64_request {
|
|
|
|
|
uint8_t operation; /* BLKIF_OP_??? */
|
2011-09-01 18:39:10 +08:00
|
|
|
union {
|
|
|
|
|
struct blkif_x86_64_request_rw rw;
|
|
|
|
|
struct blkif_x86_64_request_discard discard;
|
2013-03-08 01:32:01 +08:00
|
|
|
struct blkif_x86_64_request_other other;
|
2013-04-18 22:06:54 +08:00
|
|
|
struct blkif_x86_64_request_indirect indirect;
|
2011-09-01 18:39:10 +08:00
|
|
|
} u;
|
2011-10-13 00:12:36 +08:00
|
|
|
} __attribute__((__packed__));
|
|
|
|
|
|
2011-05-13 04:31:51 +08:00
|
|
|
DEFINE_RING_TYPES(blkif_common, struct blkif_common_request,
|
2017-06-14 04:28:27 +08:00
|
|
|
struct blkif_response);
|
2011-05-13 04:31:51 +08:00
|
|
|
DEFINE_RING_TYPES(blkif_x86_32, struct blkif_x86_32_request,
|
2017-06-14 04:28:27 +08:00
|
|
|
struct blkif_response __packed);
|
2011-05-13 04:31:51 +08:00
|
|
|
DEFINE_RING_TYPES(blkif_x86_64, struct blkif_x86_64_request,
|
2017-06-14 04:28:27 +08:00
|
|
|
struct blkif_response);
|
2011-05-13 04:31:51 +08:00
|
|
|
|
|
|
|
|
union blkif_back_rings {
|
|
|
|
|
struct blkif_back_ring native;
|
|
|
|
|
struct blkif_common_back_ring common;
|
|
|
|
|
struct blkif_x86_32_back_ring x86_32;
|
|
|
|
|
struct blkif_x86_64_back_ring x86_64;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
enum blkif_protocol {
|
|
|
|
|
BLKIF_PROTOCOL_NATIVE = 1,
|
|
|
|
|
BLKIF_PROTOCOL_X86_32 = 2,
|
|
|
|
|
BLKIF_PROTOCOL_X86_64 = 3,
|
|
|
|
|
};
|
|
|
|
|
|
2015-02-06 01:09:56 +08:00
|
|
|
/*
|
|
|
|
|
* Default protocol if the frontend doesn't specify one.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_X86
|
|
|
|
|
# define BLKIF_PROTOCOL_DEFAULT BLKIF_PROTOCOL_X86_32
|
|
|
|
|
#else
|
|
|
|
|
# define BLKIF_PROTOCOL_DEFAULT BLKIF_PROTOCOL_NATIVE
|
|
|
|
|
#endif
|
|
|
|
|
|
2011-05-13 04:53:56 +08:00
|
|
|
struct xen_vbd {
|
2011-05-12 03:57:09 +08:00
|
|
|
/* What the domain refers to this vbd as. */
|
|
|
|
|
blkif_vdev_t handle;
|
|
|
|
|
/* Non-zero -> read-only */
|
|
|
|
|
unsigned char readonly;
|
|
|
|
|
/* VDISK_xxx */
|
|
|
|
|
unsigned char type;
|
|
|
|
|
/* phys device that this vbd maps to. */
|
|
|
|
|
u32 pdevice;
|
|
|
|
|
struct block_device *bdev;
|
|
|
|
|
/* Cached size parameter. */
|
|
|
|
|
sector_t size;
|
2012-09-21 17:04:18 +08:00
|
|
|
unsigned int flush_support:1;
|
|
|
|
|
unsigned int discard_secure:1;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
unsigned int feature_gnt_persistent:1;
|
|
|
|
|
unsigned int overflow_max_grants:1;
|
2011-04-15 06:25:47 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct backend_info;
|
|
|
|
|
|
2013-04-18 02:18:59 +08:00
|
|
|
/* Number of requests that we can fit in a ring */
|
2015-06-03 13:40:01 +08:00
|
|
|
#define XEN_BLKIF_REQS_PER_PAGE 32
|
2013-04-18 02:18:59 +08:00
|
|
|
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
struct persistent_gnt {
|
|
|
|
|
struct page *page;
|
|
|
|
|
grant_ref_t gnt;
|
|
|
|
|
grant_handle_t handle;
|
2018-08-13 22:01:10 +08:00
|
|
|
unsigned long last_used;
|
2018-08-13 22:01:13 +08:00
|
|
|
bool active;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
struct rb_node node;
|
2013-04-18 02:18:57 +08:00
|
|
|
struct list_head remove_node;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
};
|
|
|
|
|
|
2015-11-14 11:12:15 +08:00
|
|
|
/* Per-ring information. */
|
|
|
|
|
struct xen_blkif_ring {
|
2011-04-15 06:25:47 +08:00
|
|
|
/* Physical parameters of the comms window. */
|
2011-05-12 03:57:09 +08:00
|
|
|
unsigned int irq;
|
|
|
|
|
union blkif_back_rings blk_rings;
|
2011-09-29 23:53:30 +08:00
|
|
|
void *blk_ring;
|
2011-04-15 06:25:47 +08:00
|
|
|
/* Private fields. */
|
2011-05-12 03:57:09 +08:00
|
|
|
spinlock_t blk_ring_lock;
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2011-05-12 03:57:09 +08:00
|
|
|
wait_queue_head_t wq;
|
2014-02-04 18:26:14 +08:00
|
|
|
atomic_t inflight;
|
2017-05-18 23:28:47 +08:00
|
|
|
bool active;
|
2015-11-14 11:12:15 +08:00
|
|
|
/* One thread per blkif ring. */
|
2011-05-12 03:57:09 +08:00
|
|
|
struct task_struct *xenblkd;
|
|
|
|
|
unsigned int waiting_reqs;
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2015-11-14 11:12:15 +08:00
|
|
|
/* List of all 'pending_req' available */
|
|
|
|
|
struct list_head pending_free;
|
|
|
|
|
/* And its spinlock. */
|
|
|
|
|
spinlock_t pending_free_lock;
|
|
|
|
|
wait_queue_head_t pending_free_wq;
|
|
|
|
|
|
2015-11-14 11:12:19 +08:00
|
|
|
/* Tree to store persistent grants. */
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
struct rb_root persistent_gnts;
|
|
|
|
|
unsigned int persistent_gnt_c;
|
2013-04-18 02:18:57 +08:00
|
|
|
atomic_t persistent_gnt_in_use;
|
|
|
|
|
unsigned long next_lru;
|
|
|
|
|
|
2015-12-09 07:44:02 +08:00
|
|
|
/* Statistics. */
|
|
|
|
|
unsigned long st_print;
|
|
|
|
|
unsigned long long st_rd_req;
|
|
|
|
|
unsigned long long st_wr_req;
|
|
|
|
|
unsigned long long st_oo_req;
|
|
|
|
|
unsigned long long st_f_req;
|
|
|
|
|
unsigned long long st_ds_req;
|
|
|
|
|
unsigned long long st_rd_sect;
|
|
|
|
|
unsigned long long st_wr_sect;
|
|
|
|
|
|
2015-11-14 11:12:19 +08:00
|
|
|
/* Used by the kworker that offload work from the persistent purge. */
|
2013-04-18 02:18:57 +08:00
|
|
|
struct list_head persistent_purge_list;
|
|
|
|
|
struct work_struct persistent_purge_work;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-25 00:58:45 +08:00
|
|
|
|
2015-11-14 11:12:19 +08:00
|
|
|
/* Buffer of free pages to map grant refs. */
|
2013-04-18 02:18:56 +08:00
|
|
|
spinlock_t free_pages_lock;
|
|
|
|
|
int free_pages_num;
|
|
|
|
|
struct list_head free_pages;
|
|
|
|
|
|
2014-05-21 04:28:50 +08:00
|
|
|
struct work_struct free_work;
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 05:54:32 +08:00
|
|
|
/* Thread shutdown wait queue. */
|
|
|
|
|
wait_queue_head_t shutdown_wq;
|
2015-11-14 11:12:15 +08:00
|
|
|
struct xen_blkif *blkif;
|
|
|
|
|
};
|
|
|
|
|
|
2011-05-13 04:47:48 +08:00
|
|
|
struct xen_blkif {
|
2011-04-15 06:25:47 +08:00
|
|
|
/* Unique identifier for this interface. */
|
2011-05-12 03:57:09 +08:00
|
|
|
domid_t domid;
|
|
|
|
|
unsigned int handle;
|
2011-04-15 06:25:47 +08:00
|
|
|
/* Comms information. */
|
2011-05-12 03:57:09 +08:00
|
|
|
enum blkif_protocol blk_protocol;
|
2011-04-15 06:25:47 +08:00
|
|
|
/* The VBD attached to this interface. */
|
2011-05-13 04:53:56 +08:00
|
|
|
struct xen_vbd vbd;
|
2011-04-15 06:25:47 +08:00
|
|
|
/* Back pointer to the backend_info. */
|
2011-05-12 03:57:09 +08:00
|
|
|
struct backend_info *be;
|
|
|
|
|
atomic_t refcnt;
|
2011-10-10 12:42:22 +08:00
|
|
|
/* for barrier (drain) requests */
|
|
|
|
|
struct completion drain_complete;
|
|
|
|
|
atomic_t drain;
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2014-05-21 04:28:50 +08:00
|
|
|
struct work_struct free_work;
|
2015-11-14 11:12:15 +08:00
|
|
|
unsigned int nr_ring_pages;
|
|
|
|
|
/* All rings for this device. */
|
2015-12-12 01:08:48 +08:00
|
|
|
struct xen_blkif_ring *rings;
|
|
|
|
|
unsigned int nr_rings;
|
xen/blkback: Squeeze page pools if a memory pressure is detected
Each `blkif` has a free pages pool for the grant mapping. The size of
the pool starts from zero and is increased on demand while processing
the I/O requests. If current I/O requests handling is finished or 100
milliseconds has passed since last I/O requests handling, it checks and
shrinks the pool to not exceed the size limit, `max_buffer_pages`.
Therefore, host administrators can cause memory pressure in blkback by
attaching a large number of block devices and inducing I/O. Such
problematic situations can be avoided by limiting the maximum number of
devices that can be attached, but finding the optimal limit is not so
easy. Improper set of the limit can results in memory pressure or a
resource underutilization. This commit avoids such problematic
situations by squeezing the pools (returns every free page in the pool
to the system) for a while (users can set this duration via a module
parameter) if memory pressure is detected.
Discussions
===========
The `blkback`'s original shrinking mechanism returns only pages in the
pool which are not currently be used by `blkback` to the system. In
other words, the pages that are not mapped with granted pages. Because
this commit is changing only the shrink limit but still uses the same
freeing mechanism it does not touch pages which are currently mapping
grants.
Once memory pressure is detected, this commit keeps the squeezing limit
for a user-specified time duration. The duration should be neither too
long nor too short. If it is too long, the squeezing incurring overhead
can reduce the I/O performance. If it is too short, `blkback` will not
free enough pages to reduce the memory pressure. This commit sets the
value as `10 milliseconds` by default because it is a short time in
terms of I/O while it is a long time in terms of memory operations.
Also, as the original shrinking mechanism works for at least every 100
milliseconds, this could be a somewhat reasonable choice. I also tested
other durations (refer to the below section for more details) and
confirmed that 10 milliseconds is the one that works best with the test.
That said, the proper duration depends on actual configurations and
workloads. That's why this commit allows users to set the duration as a
module parameter.
Memory Pressure Test
====================
To show how this commit fixes the memory pressure situation well, I
configured a test environment on a xen-running virtualization system.
On the `blkfront` running guest instances, I attach a large number of
network-backed volume devices and induce I/O to those. Meanwhile, I
measure the number of pages that swapped in (pswpin) and out (pswpout)
on the `blkback` running guest. The test ran twice, once for the
`blkback` before this commit and once for that after this commit. As
shown below, this commit has dramatically reduced the memory pressure:
pswpin pswpout
before 76,672 185,799
after 867 3,967
Optimal Aggressive Shrinking Duration
-------------------------------------
To find a best squeezing duration, I repeated the test with three
different durations (1ms, 10ms, and 100ms). The results are as below:
duration pswpin pswpout
1 707 5,095
10 867 3,967
100 362 3,348
As expected, the memory pressure decreases as the duration increases,
but the reduction become slow from the `10ms`. Based on this results, I
chose the default duration as 10ms.
Performance Overhead Test
=========================
This commit could incur I/O performance degradation under severe memory
pressure because the squeezing will require more page allocations per
I/O. To show the overhead, I artificially made a worst-case squeezing
situation and measured the I/O performance of a `blkfront` running
guest.
For the artificial squeezing, I set the `blkback.max_buffer_pages` using
the `/sys/module/xen_blkback/parameters/max_buffer_pages` file. In this
test, I set the value to `1024` and `0`. The `1024` is the default
value. Setting the value as `0` is same to a situation doing the
squeezing always (worst-case).
If the underlying block device is slow enough, the squeezing overhead
could be hidden. For the reason, I use a fast block device, namely the
rbd[1]:
# xl block-attach guest phy:/dev/ram0 xvdb w
For the I/O performance measurement, I run a simple `dd` command 5 times
directly to the device as below and collect the 'MB/s' results.
$ for i in {1..5}; do dd if=/dev/zero of=/dev/xvdb \
bs=4k count=$((256*512)); sync; done
The results are as below. 'max_pgs' represents the value of the
`blkback.max_buffer_pages` parameter.
max_pgs Min Max Median Avg Stddev
0 417 423 420 419.4 2.5099801
1024 414 425 416 417.8 4.4384682
No difference proven at 95.0% confidence
In short, even worst case squeezing on ramdisk based fast block device
makes no visible performance degradation. Please note that this is just
a very simple and minimal test. On systems using super-fast block
devices and a special I/O workload, the results might be different. If
you have any doubt, test on your machine with your workload to find the
optimal squeezing duration for you.
[1] https://www.kernel.org/doc/html/latest/admin-guide/blockdev/ramdisk.html
Reviewed-by: Roger Pau Monné <roger.pau@citrix.com>
Signed-off-by: SeongJae Park <sjpark@amazon.de>
Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
2020-01-27 16:18:10 +08:00
|
|
|
unsigned long buffer_squeeze_end;
|
2011-04-15 05:21:50 +08:00
|
|
|
};
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2013-04-18 22:06:54 +08:00
|
|
|
struct seg_buf {
|
|
|
|
|
unsigned long offset;
|
|
|
|
|
unsigned int nsec;
|
|
|
|
|
};
|
|
|
|
|
|
2013-05-02 16:21:17 +08:00
|
|
|
struct grant_page {
|
|
|
|
|
struct page *page;
|
|
|
|
|
struct persistent_gnt *persistent_gnt;
|
|
|
|
|
grant_handle_t handle;
|
|
|
|
|
grant_ref_t gref;
|
|
|
|
|
};
|
|
|
|
|
|
2013-04-18 02:18:59 +08:00
|
|
|
/*
|
|
|
|
|
* Each outstanding request that we've passed to the lower device layers has a
|
|
|
|
|
* 'pending_req' allocated to it. Each buffer_head that completes decrements
|
|
|
|
|
* the pendcnt towards zero. When it hits zero, the specified domain has a
|
|
|
|
|
* response queued for it, with the saved 'id' passed back.
|
|
|
|
|
*/
|
|
|
|
|
struct pending_req {
|
2015-11-14 11:12:15 +08:00
|
|
|
struct xen_blkif_ring *ring;
|
2013-04-18 02:18:59 +08:00
|
|
|
u64 id;
|
2015-06-17 22:28:08 +08:00
|
|
|
int nr_segs;
|
2013-04-18 02:18:59 +08:00
|
|
|
atomic_t pendcnt;
|
|
|
|
|
unsigned short operation;
|
|
|
|
|
int status;
|
|
|
|
|
struct list_head free_list;
|
2013-05-02 16:21:17 +08:00
|
|
|
struct grant_page *segments[MAX_INDIRECT_SEGMENTS];
|
2013-04-18 22:06:54 +08:00
|
|
|
/* Indirect descriptors */
|
2013-05-02 16:21:17 +08:00
|
|
|
struct grant_page *indirect_pages[MAX_INDIRECT_PAGES];
|
2013-04-18 22:06:54 +08:00
|
|
|
struct seg_buf seg[MAX_INDIRECT_SEGMENTS];
|
|
|
|
|
struct bio *biolist[MAX_INDIRECT_SEGMENTS];
|
2015-01-06 00:49:22 +08:00
|
|
|
struct gnttab_unmap_grant_ref unmap[MAX_INDIRECT_SEGMENTS];
|
|
|
|
|
struct page *unmap_pages[MAX_INDIRECT_SEGMENTS];
|
|
|
|
|
struct gntab_unmap_queue_data gnttab_unmap_data;
|
2013-04-18 02:18:59 +08:00
|
|
|
};
|
|
|
|
|
|
2011-04-20 23:21:43 +08:00
|
|
|
|
|
|
|
|
#define vbd_sz(_v) ((_v)->bdev->bd_part ? \
|
|
|
|
|
(_v)->bdev->bd_part->nr_sects : \
|
|
|
|
|
get_capacity((_v)->bdev->bd_disk))
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2011-04-20 23:50:43 +08:00
|
|
|
#define xen_blkif_get(_b) (atomic_inc(&(_b)->refcnt))
|
|
|
|
|
#define xen_blkif_put(_b) \
|
2011-04-15 06:25:47 +08:00
|
|
|
do { \
|
|
|
|
|
if (atomic_dec_and_test(&(_b)->refcnt)) \
|
2014-05-21 04:28:50 +08:00
|
|
|
schedule_work(&(_b)->free_work);\
|
2011-04-15 06:25:47 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
struct phys_req {
|
2011-05-12 03:57:09 +08:00
|
|
|
unsigned short dev;
|
2011-09-01 18:39:10 +08:00
|
|
|
blkif_sector_t nr_sects;
|
2011-05-12 03:57:09 +08:00
|
|
|
struct block_device *bdev;
|
|
|
|
|
blkif_sector_t sector_number;
|
2011-04-15 06:25:47 +08:00
|
|
|
};
|
2019-12-02 19:41:17 +08:00
|
|
|
|
2011-04-20 23:50:43 +08:00
|
|
|
int xen_blkif_interface_init(void);
|
2019-12-02 19:41:17 +08:00
|
|
|
void xen_blkif_interface_fini(void);
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2011-04-20 23:50:43 +08:00
|
|
|
int xen_blkif_xenbus_init(void);
|
2019-12-02 19:41:17 +08:00
|
|
|
void xen_blkif_xenbus_fini(void);
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2011-04-20 23:50:43 +08:00
|
|
|
irqreturn_t xen_blkif_be_int(int irq, void *dev_id);
|
|
|
|
|
int xen_blkif_schedule(void *arg);
|
2013-04-18 02:18:57 +08:00
|
|
|
int xen_blkif_purge_persistent(void *arg);
|
2015-11-14 11:12:15 +08:00
|
|
|
void xen_blkbk_free_caches(struct xen_blkif_ring *ring);
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2011-05-05 05:07:27 +08:00
|
|
|
int xen_blkbk_flush_diskcache(struct xenbus_transaction xbt,
|
|
|
|
|
struct backend_info *be, int state);
|
2011-04-15 06:25:47 +08:00
|
|
|
|
2011-10-10 12:42:22 +08:00
|
|
|
int xen_blkbk_barrier(struct xenbus_transaction xbt,
|
|
|
|
|
struct backend_info *be, int state);
|
2011-04-20 23:50:43 +08:00
|
|
|
struct xenbus_device *xen_blkbk_xenbus(struct backend_info *be);
|
2014-02-12 11:34:03 +08:00
|
|
|
void xen_blkbk_unmap_purged_grants(struct work_struct *work);
|
2010-03-19 06:35:05 +08:00
|
|
|
|
2011-05-13 04:23:06 +08:00
|
|
|
static inline void blkif_get_x86_32_req(struct blkif_request *dst,
|
2011-05-12 04:23:39 +08:00
|
|
|
struct blkif_x86_32_request *src)
|
|
|
|
|
{
|
2013-04-18 22:06:54 +08:00
|
|
|
int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST, j;
|
2015-11-04 00:34:09 +08:00
|
|
|
dst->operation = READ_ONCE(src->operation);
|
|
|
|
|
switch (dst->operation) {
|
2011-09-01 18:39:10 +08:00
|
|
|
case BLKIF_OP_READ:
|
|
|
|
|
case BLKIF_OP_WRITE:
|
|
|
|
|
case BLKIF_OP_WRITE_BARRIER:
|
|
|
|
|
case BLKIF_OP_FLUSH_DISKCACHE:
|
2011-10-13 00:12:36 +08:00
|
|
|
dst->u.rw.nr_segments = src->u.rw.nr_segments;
|
|
|
|
|
dst->u.rw.handle = src->u.rw.handle;
|
|
|
|
|
dst->u.rw.id = src->u.rw.id;
|
2011-09-01 18:39:10 +08:00
|
|
|
dst->u.rw.sector_number = src->u.rw.sector_number;
|
|
|
|
|
barrier();
|
2011-10-13 00:12:36 +08:00
|
|
|
if (n > dst->u.rw.nr_segments)
|
|
|
|
|
n = dst->u.rw.nr_segments;
|
2011-09-01 18:39:10 +08:00
|
|
|
for (i = 0; i < n; i++)
|
|
|
|
|
dst->u.rw.seg[i] = src->u.rw.seg[i];
|
|
|
|
|
break;
|
|
|
|
|
case BLKIF_OP_DISCARD:
|
2011-10-13 04:23:30 +08:00
|
|
|
dst->u.discard.flag = src->u.discard.flag;
|
2012-05-26 04:11:09 +08:00
|
|
|
dst->u.discard.id = src->u.discard.id;
|
2011-09-01 18:39:10 +08:00
|
|
|
dst->u.discard.sector_number = src->u.discard.sector_number;
|
|
|
|
|
dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
|
|
|
|
|
break;
|
2013-04-18 22:06:54 +08:00
|
|
|
case BLKIF_OP_INDIRECT:
|
|
|
|
|
dst->u.indirect.indirect_op = src->u.indirect.indirect_op;
|
|
|
|
|
dst->u.indirect.nr_segments = src->u.indirect.nr_segments;
|
|
|
|
|
dst->u.indirect.handle = src->u.indirect.handle;
|
|
|
|
|
dst->u.indirect.id = src->u.indirect.id;
|
|
|
|
|
dst->u.indirect.sector_number = src->u.indirect.sector_number;
|
|
|
|
|
barrier();
|
|
|
|
|
j = min(MAX_INDIRECT_PAGES, INDIRECT_PAGES(dst->u.indirect.nr_segments));
|
|
|
|
|
for (i = 0; i < j; i++)
|
|
|
|
|
dst->u.indirect.indirect_grefs[i] =
|
|
|
|
|
src->u.indirect.indirect_grefs[i];
|
|
|
|
|
break;
|
2011-09-01 18:39:10 +08:00
|
|
|
default:
|
2013-03-08 01:32:01 +08:00
|
|
|
/*
|
|
|
|
|
* Don't know how to translate this op. Only get the
|
|
|
|
|
* ID so failure can be reported to the frontend.
|
|
|
|
|
*/
|
|
|
|
|
dst->u.other.id = src->u.other.id;
|
2011-09-01 18:39:10 +08:00
|
|
|
break;
|
|
|
|
|
}
|
2011-05-12 04:23:39 +08:00
|
|
|
}
|
|
|
|
|
|
2011-05-13 04:23:06 +08:00
|
|
|
static inline void blkif_get_x86_64_req(struct blkif_request *dst,
|
2011-05-12 04:23:39 +08:00
|
|
|
struct blkif_x86_64_request *src)
|
|
|
|
|
{
|
2013-04-18 22:06:54 +08:00
|
|
|
int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST, j;
|
2015-11-04 00:34:09 +08:00
|
|
|
dst->operation = READ_ONCE(src->operation);
|
|
|
|
|
switch (dst->operation) {
|
2011-09-01 18:39:10 +08:00
|
|
|
case BLKIF_OP_READ:
|
|
|
|
|
case BLKIF_OP_WRITE:
|
|
|
|
|
case BLKIF_OP_WRITE_BARRIER:
|
|
|
|
|
case BLKIF_OP_FLUSH_DISKCACHE:
|
2011-10-13 00:12:36 +08:00
|
|
|
dst->u.rw.nr_segments = src->u.rw.nr_segments;
|
|
|
|
|
dst->u.rw.handle = src->u.rw.handle;
|
|
|
|
|
dst->u.rw.id = src->u.rw.id;
|
2011-09-01 18:39:10 +08:00
|
|
|
dst->u.rw.sector_number = src->u.rw.sector_number;
|
|
|
|
|
barrier();
|
2011-10-13 00:12:36 +08:00
|
|
|
if (n > dst->u.rw.nr_segments)
|
|
|
|
|
n = dst->u.rw.nr_segments;
|
2011-09-01 18:39:10 +08:00
|
|
|
for (i = 0; i < n; i++)
|
|
|
|
|
dst->u.rw.seg[i] = src->u.rw.seg[i];
|
|
|
|
|
break;
|
|
|
|
|
case BLKIF_OP_DISCARD:
|
2011-10-13 04:23:30 +08:00
|
|
|
dst->u.discard.flag = src->u.discard.flag;
|
2012-05-26 04:11:09 +08:00
|
|
|
dst->u.discard.id = src->u.discard.id;
|
2011-09-01 18:39:10 +08:00
|
|
|
dst->u.discard.sector_number = src->u.discard.sector_number;
|
|
|
|
|
dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
|
|
|
|
|
break;
|
2013-04-18 22:06:54 +08:00
|
|
|
case BLKIF_OP_INDIRECT:
|
|
|
|
|
dst->u.indirect.indirect_op = src->u.indirect.indirect_op;
|
|
|
|
|
dst->u.indirect.nr_segments = src->u.indirect.nr_segments;
|
|
|
|
|
dst->u.indirect.handle = src->u.indirect.handle;
|
|
|
|
|
dst->u.indirect.id = src->u.indirect.id;
|
|
|
|
|
dst->u.indirect.sector_number = src->u.indirect.sector_number;
|
|
|
|
|
barrier();
|
|
|
|
|
j = min(MAX_INDIRECT_PAGES, INDIRECT_PAGES(dst->u.indirect.nr_segments));
|
|
|
|
|
for (i = 0; i < j; i++)
|
|
|
|
|
dst->u.indirect.indirect_grefs[i] =
|
|
|
|
|
src->u.indirect.indirect_grefs[i];
|
|
|
|
|
break;
|
2011-09-01 18:39:10 +08:00
|
|
|
default:
|
2013-03-08 01:32:01 +08:00
|
|
|
/*
|
|
|
|
|
* Don't know how to translate this op. Only get the
|
|
|
|
|
* ID so failure can be reported to the frontend.
|
|
|
|
|
*/
|
|
|
|
|
dst->u.other.id = src->u.other.id;
|
2011-09-01 18:39:10 +08:00
|
|
|
break;
|
|
|
|
|
}
|
2011-05-12 04:23:39 +08:00
|
|
|
}
|
|
|
|
|
|
2011-05-13 04:58:21 +08:00
|
|
|
#endif /* __XEN_BLKIF__BACKEND__COMMON_H__ */
|