UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'stable/for-jens-3.8' of git://git.kernel.org/pub/scm/linux/kernel/git/konrad/xen into for-3.8/drivers

This commit is contained in:
Jens Axboe 2012-11-12 09:18:47 -07:00
parent f1d6a328bb cb5bd4d19b
commit 8d0ff3924b
4 changed files with 475 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

289
drivers/block/xen-blkback/blkback.c
View File

@ -39,6 +39,7 @@
#include <linux/list.h> #include <linux/list.h>
#include <linux/delay.h> #include <linux/delay.h>
#include <linux/freezer.h> #include <linux/freezer.h>
#include <linux/bitmap.h>
#include <xen/events.h> #include <xen/events.h>
#include <xen/page.h> #include <xen/page.h>
@ -79,6 +80,7 @@ struct pending_req {
unsigned short operation; unsigned short operation;
int status; int status;
struct list_head free_list; struct list_head free_list;
DECLARE_BITMAP(unmap_seg, BLKIF_MAX_SEGMENTS_PER_REQUEST);
}; };
#define BLKBACK_INVALID_HANDLE (~0) #define BLKBACK_INVALID_HANDLE (~0)
@ -98,6 +100,36 @@ struct xen_blkbk {
static struct xen_blkbk *blkbk; static struct xen_blkbk *blkbk;
/*
* Maximum number of grant pages that can be mapped in blkback.
* BLKIF_MAX_SEGMENTS_PER_REQUEST * RING_SIZE is the maximum number of
* pages that blkback will persistently map.
* Currently, this is:
* RING_SIZE = 32 (for all known ring types)
* BLKIF_MAX_SEGMENTS_PER_REQUEST = 11
* sizeof(struct persistent_gnt) = 48
* So the maximum memory used to store the grants is:
* 32 * 11 * 48 = 16896 bytes
*/
static inline unsigned int max_mapped_grant_pages(enum blkif_protocol protocol)
{
switch (protocol) {
case BLKIF_PROTOCOL_NATIVE:
return __CONST_RING_SIZE(blkif, PAGE_SIZE) *
BLKIF_MAX_SEGMENTS_PER_REQUEST;
case BLKIF_PROTOCOL_X86_32:
return __CONST_RING_SIZE(blkif_x86_32, PAGE_SIZE) *
BLKIF_MAX_SEGMENTS_PER_REQUEST;
case BLKIF_PROTOCOL_X86_64:
return __CONST_RING_SIZE(blkif_x86_64, PAGE_SIZE) *
BLKIF_MAX_SEGMENTS_PER_REQUEST;
default:
BUG();
}
return 0;
}
/* /*
* Little helpful macro to figure out the index and virtual address of the * Little helpful macro to figure out the index and virtual address of the
* pending_pages[..]. For each 'pending_req' we have have up to * pending_pages[..]. For each 'pending_req' we have have up to
@ -129,6 +161,57 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
static void make_response(struct xen_blkif *blkif, u64 id, static void make_response(struct xen_blkif *blkif, u64 id,
unsigned short op, int st); unsigned short op, int st);
#define foreach_grant(pos, rbtree, node) \
for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node); \
&(pos)->node != NULL; \
(pos) = container_of(rb_next(&(pos)->node), typeof(*(pos)), node))
static void add_persistent_gnt(struct rb_root *root,
struct persistent_gnt *persistent_gnt)
{
struct rb_node **new = &(root->rb_node), *parent = NULL;
struct persistent_gnt *this;
/* Figure out where to put new node */
while (*new) {
this = container_of(*new, struct persistent_gnt, node);
parent = *new;
if (persistent_gnt->gnt < this->gnt)
new = &((*new)->rb_left);
else if (persistent_gnt->gnt > this->gnt)
new = &((*new)->rb_right);
else {
pr_alert(DRV_PFX " trying to add a gref that's already in the tree\n");
BUG();
}
}
/* Add new node and rebalance tree. */
rb_link_node(&(persistent_gnt->node), parent, new);
rb_insert_color(&(persistent_gnt->node), root);
}
static struct persistent_gnt *get_persistent_gnt(struct rb_root *root,
grant_ref_t gref)
{
struct persistent_gnt *data;
struct rb_node *node = root->rb_node;
while (node) {
data = container_of(node, struct persistent_gnt, node);
if (gref < data->gnt)
node = node->rb_left;
else if (gref > data->gnt)
node = node->rb_right;
else
return data;
}
return NULL;
}
/* /*
* Retrieve from the 'pending_reqs' a free pending_req structure to be used. * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
*/ */
@ -275,6 +358,11 @@ int xen_blkif_schedule(void *arg)
{ {
struct xen_blkif *blkif = arg; struct xen_blkif *blkif = arg;
struct xen_vbd *vbd = &blkif->vbd; struct xen_vbd *vbd = &blkif->vbd;
struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
struct persistent_gnt *persistent_gnt;
int ret = 0;
int segs_to_unmap = 0;
xen_blkif_get(blkif); xen_blkif_get(blkif);
@ -302,6 +390,36 @@ int xen_blkif_schedule(void *arg)
print_stats(blkif); print_stats(blkif);
} }
/* Free all persistent grant pages */
if (!RB_EMPTY_ROOT(&blkif->persistent_gnts)) {
foreach_grant(persistent_gnt, &blkif->persistent_gnts, node) {
BUG_ON(persistent_gnt->handle ==
BLKBACK_INVALID_HANDLE);
gnttab_set_unmap_op(&unmap[segs_to_unmap],
(unsigned long) pfn_to_kaddr(page_to_pfn(
persistent_gnt->page)),
GNTMAP_host_map,
persistent_gnt->handle);
pages[segs_to_unmap] = persistent_gnt->page;
rb_erase(&persistent_gnt->node,
&blkif->persistent_gnts);
kfree(persistent_gnt);
blkif->persistent_gnt_c--;
if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
!rb_next(&persistent_gnt->node)) {
ret = gnttab_unmap_refs(unmap, NULL, pages,
segs_to_unmap);
BUG_ON(ret);
segs_to_unmap = 0;
}
}
}
BUG_ON(blkif->persistent_gnt_c != 0);
BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts));
if (log_stats) if (log_stats)
print_stats(blkif); print_stats(blkif);
@ -328,6 +446,8 @@ static void xen_blkbk_unmap(struct pending_req *req)
int ret; int ret;
for (i = 0; i < req->nr_pages; i++) { for (i = 0; i < req->nr_pages; i++) {
if (!test_bit(i, req->unmap_seg))
continue;
handle = pending_handle(req, i); handle = pending_handle(req, i);
if (handle == BLKBACK_INVALID_HANDLE) if (handle == BLKBACK_INVALID_HANDLE)
continue; continue;
@ -344,12 +464,26 @@ static void xen_blkbk_unmap(struct pending_req *req)
static int xen_blkbk_map(struct blkif_request *req, static int xen_blkbk_map(struct blkif_request *req,
struct pending_req *pending_req, struct pending_req *pending_req,
struct seg_buf seg[]) struct seg_buf seg[],
struct page *pages[])
{ {
struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST]; struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
int i; struct persistent_gnt *persistent_gnts[BLKIF_MAX_SEGMENTS_PER_REQUEST];
struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST];
struct persistent_gnt *persistent_gnt = NULL;
struct xen_blkif *blkif = pending_req->blkif;
phys_addr_t addr = 0;
int i, j;
bool new_map;
int nseg = req->u.rw.nr_segments; int nseg = req->u.rw.nr_segments;
int segs_to_map = 0;
int ret = 0; int ret = 0;
int use_persistent_gnts;
use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
BUG_ON(blkif->persistent_gnt_c >
max_mapped_grant_pages(pending_req->blkif->blk_protocol));
/* /*
* Fill out preq.nr_sects with proper amount of sectors, and setup * Fill out preq.nr_sects with proper amount of sectors, and setup
@ -359,36 +493,146 @@ static int xen_blkbk_map(struct blkif_request *req,
for (i = 0; i < nseg; i++) { for (i = 0; i < nseg; i++) {
uint32_t flags; uint32_t flags;
flags = GNTMAP_host_map; if (use_persistent_gnts)
if (pending_req->operation != BLKIF_OP_READ) persistent_gnt = get_persistent_gnt(
flags |= GNTMAP_readonly; &blkif->persistent_gnts,
gnttab_set_map_op(&map[i], vaddr(pending_req, i), flags, req->u.rw.seg[i].gref);
req->u.rw.seg[i].gref,
pending_req->blkif->domid); if (persistent_gnt) {
/*
* We are using persistent grants and
* the grant is already mapped
*/
new_map = false;
} else if (use_persistent_gnts &&
blkif->persistent_gnt_c <
max_mapped_grant_pages(blkif->blk_protocol)) {
/*
* We are using persistent grants, the grant is
* not mapped but we have room for it
*/
new_map = true;
persistent_gnt = kmalloc(
sizeof(struct persistent_gnt),
GFP_KERNEL);
if (!persistent_gnt)
return -ENOMEM;
persistent_gnt->page = alloc_page(GFP_KERNEL);
if (!persistent_gnt->page) {
kfree(persistent_gnt);
return -ENOMEM;
}
persistent_gnt->gnt = req->u.rw.seg[i].gref;
persistent_gnt->handle = BLKBACK_INVALID_HANDLE;
pages_to_gnt[segs_to_map] =
persistent_gnt->page;
addr = (unsigned long) pfn_to_kaddr(
page_to_pfn(persistent_gnt->page));
add_persistent_gnt(&blkif->persistent_gnts,
persistent_gnt);
blkif->persistent_gnt_c++;
pr_debug(DRV_PFX " grant %u added to the tree of persistent grants, using %u/%u\n",
persistent_gnt->gnt, blkif->persistent_gnt_c,
max_mapped_grant_pages(blkif->blk_protocol));
} else {
/*
* We are either using persistent grants and
* hit the maximum limit of grants mapped,
* or we are not using persistent grants.
*/
if (use_persistent_gnts &&
!blkif->vbd.overflow_max_grants) {
blkif->vbd.overflow_max_grants = 1;
pr_alert(DRV_PFX " domain %u, device %#x is using maximum number of persistent grants\n",
blkif->domid, blkif->vbd.handle);
}
new_map = true;
pages[i] = blkbk->pending_page(pending_req, i);
addr = vaddr(pending_req, i);
pages_to_gnt[segs_to_map] =
blkbk->pending_page(pending_req, i);
}
if (persistent_gnt) {
pages[i] = persistent_gnt->page;
persistent_gnts[i] = persistent_gnt;
} else {
persistent_gnts[i] = NULL;
}
if (new_map) {
flags = GNTMAP_host_map;
if (!persistent_gnt &&
(pending_req->operation != BLKIF_OP_READ))
flags |= GNTMAP_readonly;
gnttab_set_map_op(&map[segs_to_map++], addr,
flags, req->u.rw.seg[i].gref,
blkif->domid);
}
} }
ret = gnttab_map_refs(map, NULL, &blkbk->pending_page(pending_req, 0), nseg); if (segs_to_map) {
BUG_ON(ret); ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map);
BUG_ON(ret);
}
/* /*
* Now swizzle the MFN in our domain with the MFN from the other domain * Now swizzle the MFN in our domain with the MFN from the other domain
* so that when we access vaddr(pending_req,i) it has the contents of * so that when we access vaddr(pending_req,i) it has the contents of
* the page from the other domain. * the page from the other domain.
*/ */
for (i = 0; i < nseg; i++) { bitmap_zero(pending_req->unmap_seg, BLKIF_MAX_SEGMENTS_PER_REQUEST);
if (unlikely(map[i].status != 0)) { for (i = 0, j = 0; i < nseg; i++) {
pr_debug(DRV_PFX "invalid buffer -- could not remap it\n"); if (!persistent_gnts[i] ||
map[i].handle = BLKBACK_INVALID_HANDLE; persistent_gnts[i]->handle == BLKBACK_INVALID_HANDLE) {
ret |= 1; /* This is a newly mapped grant */
BUG_ON(j >= segs_to_map);
if (unlikely(map[j].status != 0)) {
pr_debug(DRV_PFX "invalid buffer -- could not remap it\n");
map[j].handle = BLKBACK_INVALID_HANDLE;
ret |= 1;
if (persistent_gnts[i]) {
rb_erase(&persistent_gnts[i]->node,
&blkif->persistent_gnts);
blkif->persistent_gnt_c--;
kfree(persistent_gnts[i]);
persistent_gnts[i] = NULL;
}
}
} }
if (persistent_gnts[i]) {
if (persistent_gnts[i]->handle ==
BLKBACK_INVALID_HANDLE) {
/*
* If this is a new persistent grant
* save the handler
*/
persistent_gnts[i]->handle = map[j].handle;
persistent_gnts[i]->dev_bus_addr =
map[j++].dev_bus_addr;
}
pending_handle(pending_req, i) =
persistent_gnts[i]->handle;
pending_handle(pending_req, i) = map[i].handle; if (ret)
continue;
if (ret) seg[i].buf = persistent_gnts[i]->dev_bus_addr |
continue; (req->u.rw.seg[i].first_sect << 9);
} else {
pending_handle(pending_req, i) = map[j].handle;
bitmap_set(pending_req->unmap_seg, i, 1);
seg[i].buf = map[i].dev_bus_addr | if (ret) {
(req->u.rw.seg[i].first_sect << 9); j++;
continue;
}
seg[i].buf = map[j++].dev_bus_addr |
(req->u.rw.seg[i].first_sect << 9);
}
} }
return ret; return ret;
} }
@ -591,6 +835,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
int operation; int operation;
struct blk_plug plug; struct blk_plug plug;
bool drain = false; bool drain = false;
struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
switch (req->operation) { switch (req->operation) {
case BLKIF_OP_READ: case BLKIF_OP_READ:
@ -677,7 +922,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
* the hypercall to unmap the grants - that is all done in * the hypercall to unmap the grants - that is all done in
* xen_blkbk_unmap. * xen_blkbk_unmap.
*/ */
if (xen_blkbk_map(req, pending_req, seg)) if (xen_blkbk_map(req, pending_req, seg, pages))
goto fail_flush; goto fail_flush;
/* /*
@ -689,7 +934,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
for (i = 0; i < nseg; i++) { for (i = 0; i < nseg; i++) {
while ((bio == NULL) || while ((bio == NULL) ||
(bio_add_page(bio, (bio_add_page(bio,
blkbk->pending_page(pending_req, i), pages[i],
seg[i].nsec << 9, seg[i].nsec << 9,
seg[i].buf & ~PAGE_MASK) == 0)) { seg[i].buf & ~PAGE_MASK) == 0)) {

17
drivers/block/xen-blkback/common.h
View File

@ -34,6 +34,7 @@
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <linux/wait.h> #include <linux/wait.h>
#include <linux/io.h> #include <linux/io.h>
#include <linux/rbtree.h>
#include <asm/setup.h> #include <asm/setup.h>
#include <asm/pgalloc.h> #include <asm/pgalloc.h>
#include <asm/hypervisor.h> #include <asm/hypervisor.h>
@ -160,10 +161,22 @@ struct xen_vbd {
sector_t size; sector_t size;
bool flush_support; bool flush_support;
bool discard_secure; bool discard_secure;
unsigned int feature_gnt_persistent:1;
unsigned int overflow_max_grants:1;
}; };
struct backend_info; struct backend_info;
struct persistent_gnt {
struct page *page;
grant_ref_t gnt;
grant_handle_t handle;
uint64_t dev_bus_addr;
struct rb_node node;
};
struct xen_blkif { struct xen_blkif {
/* Unique identifier for this interface. */ /* Unique identifier for this interface. */
domid_t domid; domid_t domid;
@ -190,6 +203,10 @@ struct xen_blkif {
struct task_struct *xenblkd; struct task_struct *xenblkd;
unsigned int waiting_reqs; unsigned int waiting_reqs;
/* tree to store persistent grants */
struct rb_root persistent_gnts;
unsigned int persistent_gnt_c;
/* statistics */ /* statistics */
unsigned long st_print; unsigned long st_print;
int st_rd_req; int st_rd_req;

23
drivers/block/xen-blkback/xenbus.c
View File

@ -118,6 +118,7 @@ static struct xen_blkif *xen_blkif_alloc(domid_t domid)
atomic_set(&blkif->drain, 0); atomic_set(&blkif->drain, 0);
blkif->st_print = jiffies; blkif->st_print = jiffies;
init_waitqueue_head(&blkif->waiting_to_free); init_waitqueue_head(&blkif->waiting_to_free);
blkif->persistent_gnts.rb_node = NULL;
return blkif; return blkif;
} }
@ -673,6 +674,13 @@ again:
xen_blkbk_barrier(xbt, be, be->blkif->vbd.flush_support); xen_blkbk_barrier(xbt, be, be->blkif->vbd.flush_support);
err = xenbus_printf(xbt, dev->nodename, "feature-persistent", "%u", 1);
if (err) {
xenbus_dev_fatal(dev, err, "writing %s/feature-persistent",
dev->nodename);
goto abort;
}
err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu", err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
(unsigned long long)vbd_sz(&be->blkif->vbd)); (unsigned long long)vbd_sz(&be->blkif->vbd));
if (err) { if (err) {
@ -721,6 +729,7 @@ static int connect_ring(struct backend_info *be)
struct xenbus_device *dev = be->dev; struct xenbus_device *dev = be->dev;
unsigned long ring_ref; unsigned long ring_ref;
unsigned int evtchn; unsigned int evtchn;
unsigned int pers_grants;
char protocol[64] = ""; char protocol[64] = "";
int err; int err;
@ -750,8 +759,18 @@ static int connect_ring(struct backend_info *be)
xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol); xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol);
return -1; return -1;
} }
pr_info(DRV_PFX "ring-ref %ld, event-channel %d, protocol %d (%s)\n", err = xenbus_gather(XBT_NIL, dev->otherend,
ring_ref, evtchn, be->blkif->blk_protocol, protocol); "feature-persistent", "%u",
&pers_grants, NULL);
if (err)
pers_grants = 0;
be->blkif->vbd.feature_gnt_persistent = pers_grants;
be->blkif->vbd.overflow_max_grants = 0;
pr_info(DRV_PFX "ring-ref %ld, event-channel %d, protocol %d (%s) %s\n",
ring_ref, evtchn, be->blkif->blk_protocol, protocol,
pers_grants ? "persistent grants" : "");
/* Map the shared frame, irq etc. */ /* Map the shared frame, irq etc. */
err = xen_blkif_map(be->blkif, ring_ref, evtchn); err = xen_blkif_map(be->blkif, ring_ref, evtchn);

198
drivers/block/xen-blkfront.c
View File

@ -44,6 +44,7 @@
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/bitmap.h> #include <linux/bitmap.h>
#include <linux/llist.h>
#include <xen/xen.h> #include <xen/xen.h>
#include <xen/xenbus.h> #include <xen/xenbus.h>
@ -64,10 +65,17 @@ enum blkif_state {
BLKIF_STATE_SUSPENDED, BLKIF_STATE_SUSPENDED,
}; };
struct grant {
grant_ref_t gref;
unsigned long pfn;
struct llist_node node;
};
struct blk_shadow { struct blk_shadow {
struct blkif_request req; struct blkif_request req;
struct request *request; struct request *request;
unsigned long frame[BLKIF_MAX_SEGMENTS_PER_REQUEST]; unsigned long frame[BLKIF_MAX_SEGMENTS_PER_REQUEST];
struct grant *grants_used[BLKIF_MAX_SEGMENTS_PER_REQUEST];
}; };
static DEFINE_MUTEX(blkfront_mutex); static DEFINE_MUTEX(blkfront_mutex);
@ -97,6 +105,8 @@ struct blkfront_info
struct work_struct work; struct work_struct work;
struct gnttab_free_callback callback; struct gnttab_free_callback callback;
struct blk_shadow shadow[BLK_RING_SIZE]; struct blk_shadow shadow[BLK_RING_SIZE];
struct llist_head persistent_gnts;
unsigned int persistent_gnts_c;
unsigned long shadow_free; unsigned long shadow_free;
unsigned int feature_flush; unsigned int feature_flush;
unsigned int flush_op; unsigned int flush_op;
@ -104,6 +114,7 @@ struct blkfront_info
unsigned int feature_secdiscard:1; unsigned int feature_secdiscard:1;
unsigned int discard_granularity; unsigned int discard_granularity;
unsigned int discard_alignment; unsigned int discard_alignment;
unsigned int feature_persistent:1;
int is_ready; int is_ready;
}; };
@ -287,21 +298,36 @@ static int blkif_queue_request(struct request *req)
unsigned long id; unsigned long id;
unsigned int fsect, lsect; unsigned int fsect, lsect;
int i, ref; int i, ref;
/*
* Used to store if we are able to queue the request by just using
* existing persistent grants, or if we have to get new grants,
* as there are not sufficiently many free.
*/
bool new_persistent_gnts;
grant_ref_t gref_head; grant_ref_t gref_head;
struct page *granted_page;
struct grant *gnt_list_entry = NULL;
struct scatterlist *sg; struct scatterlist *sg;
if (unlikely(info->connected != BLKIF_STATE_CONNECTED)) if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
return 1; return 1;
if (gnttab_alloc_grant_references( /* Check if we have enought grants to allocate a requests */
BLKIF_MAX_SEGMENTS_PER_REQUEST, &gref_head) < 0) { if (info->persistent_gnts_c < BLKIF_MAX_SEGMENTS_PER_REQUEST) {
gnttab_request_free_callback( new_persistent_gnts = 1;
&info->callback, if (gnttab_alloc_grant_references(
blkif_restart_queue_callback, BLKIF_MAX_SEGMENTS_PER_REQUEST - info->persistent_gnts_c,
info, &gref_head) < 0) {
BLKIF_MAX_SEGMENTS_PER_REQUEST); gnttab_request_free_callback(
return 1; &info->callback,
} blkif_restart_queue_callback,
info,
BLKIF_MAX_SEGMENTS_PER_REQUEST);
return 1;
}
} else
new_persistent_gnts = 0;
/* Fill out a communications ring structure. */ /* Fill out a communications ring structure. */
ring_req = RING_GET_REQUEST(&info->ring, info->ring.req_prod_pvt); ring_req = RING_GET_REQUEST(&info->ring, info->ring.req_prod_pvt);
@ -341,18 +367,73 @@ static int blkif_queue_request(struct request *req)
BLKIF_MAX_SEGMENTS_PER_REQUEST); BLKIF_MAX_SEGMENTS_PER_REQUEST);
for_each_sg(info->sg, sg, ring_req->u.rw.nr_segments, i) { for_each_sg(info->sg, sg, ring_req->u.rw.nr_segments, i) {
buffer_mfn = pfn_to_mfn(page_to_pfn(sg_page(sg)));
fsect = sg->offset >> 9; fsect = sg->offset >> 9;
lsect = fsect + (sg->length >> 9) - 1; lsect = fsect + (sg->length >> 9) - 1;
/* install a grant reference. */
ref = gnttab_claim_grant_reference(&gref_head);
BUG_ON(ref == -ENOSPC);
gnttab_grant_foreign_access_ref( if (info->persistent_gnts_c) {
ref, BUG_ON(llist_empty(&info->persistent_gnts));
gnt_list_entry = llist_entry(
llist_del_first(&info->persistent_gnts),
struct grant, node);
ref = gnt_list_entry->gref;
buffer_mfn = pfn_to_mfn(gnt_list_entry->pfn);
info->persistent_gnts_c--;
} else {
ref = gnttab_claim_grant_reference(&gref_head);
BUG_ON(ref == -ENOSPC);
gnt_list_entry =
kmalloc(sizeof(struct grant),
GFP_ATOMIC);
if (!gnt_list_entry)
return -ENOMEM;
granted_page = alloc_page(GFP_ATOMIC);
if (!granted_page) {
kfree(gnt_list_entry);
return -ENOMEM;
}
gnt_list_entry->pfn =
page_to_pfn(granted_page);
gnt_list_entry->gref = ref;
buffer_mfn = pfn_to_mfn(page_to_pfn(
granted_page));
gnttab_grant_foreign_access_ref(ref,
info->xbdev->otherend_id, info->xbdev->otherend_id,
buffer_mfn, buffer_mfn, 0);
rq_data_dir(req)); }
info->shadow[id].grants_used[i] = gnt_list_entry;
if (rq_data_dir(req)) {
char *bvec_data;
void *shared_data;
BUG_ON(sg->offset + sg->length > PAGE_SIZE);
shared_data = kmap_atomic(
pfn_to_page(gnt_list_entry->pfn));
bvec_data = kmap_atomic(sg_page(sg));
/*
* this does not wipe data stored outside the
* range sg->offset..sg->offset+sg->length.
* Therefore, blkback *could* see data from
* previous requests. This is OK as long as
* persistent grants are shared with just one
* domain. It may need refactoring if this
* changes
*/
memcpy(shared_data + sg->offset,
bvec_data + sg->offset,
sg->length);
kunmap_atomic(bvec_data);
kunmap_atomic(shared_data);
}
info->shadow[id].frame[i] = mfn_to_pfn(buffer_mfn); info->shadow[id].frame[i] = mfn_to_pfn(buffer_mfn);
ring_req->u.rw.seg[i] = ring_req->u.rw.seg[i] =
@ -368,7 +449,8 @@ static int blkif_queue_request(struct request *req)
/* Keep a private copy so we can reissue requests when recovering. */ /* Keep a private copy so we can reissue requests when recovering. */
info->shadow[id].req = *ring_req; info->shadow[id].req = *ring_req;
gnttab_free_grant_references(gref_head); if (new_persistent_gnts)
gnttab_free_grant_references(gref_head);
return 0; return 0;
} }
@ -480,12 +562,13 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size)
static void xlvbd_flush(struct blkfront_info *info) static void xlvbd_flush(struct blkfront_info *info)
{ {
blk_queue_flush(info->rq, info->feature_flush); blk_queue_flush(info->rq, info->feature_flush);
printk(KERN_INFO "blkfront: %s: %s: %s\n", printk(KERN_INFO "blkfront: %s: %s: %s %s\n",
info->gd->disk_name, info->gd->disk_name,
info->flush_op == BLKIF_OP_WRITE_BARRIER ? info->flush_op == BLKIF_OP_WRITE_BARRIER ?
"barrier" : (info->flush_op == BLKIF_OP_FLUSH_DISKCACHE ? "barrier" : (info->flush_op == BLKIF_OP_FLUSH_DISKCACHE ?
"flush diskcache" : "barrier or flush"), "flush diskcache" : "barrier or flush"),
info->feature_flush ? "enabled" : "disabled"); info->feature_flush ? "enabled" : "disabled",
info->feature_persistent ? "using persistent grants" : "");
} }
static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset) static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
@ -707,6 +790,9 @@ static void blkif_restart_queue(struct work_struct *work)
static void blkif_free(struct blkfront_info *info, int suspend) static void blkif_free(struct blkfront_info *info, int suspend)
{ {
struct llist_node *all_gnts;
struct grant *persistent_gnt;
/* Prevent new requests being issued until we fix things up. */ /* Prevent new requests being issued until we fix things up. */
spin_lock_irq(&info->io_lock); spin_lock_irq(&info->io_lock);
info->connected = suspend ? info->connected = suspend ?
@ -714,6 +800,17 @@ static void blkif_free(struct blkfront_info *info, int suspend)
/* No more blkif_request(). */ /* No more blkif_request(). */
if (info->rq) if (info->rq)
blk_stop_queue(info->rq); blk_stop_queue(info->rq);
/* Remove all persistent grants */
if (info->persistent_gnts_c) {
all_gnts = llist_del_all(&info->persistent_gnts);
llist_for_each_entry(persistent_gnt, all_gnts, node) {
gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
kfree(persistent_gnt);
}
info->persistent_gnts_c = 0;
}
/* No more gnttab callback work. */ /* No more gnttab callback work. */
gnttab_cancel_free_callback(&info->callback); gnttab_cancel_free_callback(&info->callback);
spin_unlock_irq(&info->io_lock); spin_unlock_irq(&info->io_lock);
@ -734,13 +831,43 @@ static void blkif_free(struct blkfront_info *info, int suspend)
} }
static void blkif_completion(struct blk_shadow *s) static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
struct blkif_response *bret)
{ {
int i; int i;
/* Do not let BLKIF_OP_DISCARD as nr_segment is in the same place struct bio_vec *bvec;
* flag. */ struct req_iterator iter;
for (i = 0; i < s->req.u.rw.nr_segments; i++) unsigned long flags;
gnttab_end_foreign_access(s->req.u.rw.seg[i].gref, 0, 0UL); char *bvec_data;
void *shared_data;
unsigned int offset = 0;
if (bret->operation == BLKIF_OP_READ) {
/*
* Copy the data received from the backend into the bvec.
* Since bv_offset can be different than 0, and bv_len different
* than PAGE_SIZE, we have to keep track of the current offset,
* to be sure we are copying the data from the right shared page.
*/
rq_for_each_segment(bvec, s->request, iter) {
BUG_ON((bvec->bv_offset + bvec->bv_len) > PAGE_SIZE);
i = offset >> PAGE_SHIFT;
BUG_ON(i >= s->req.u.rw.nr_segments);
shared_data = kmap_atomic(
pfn_to_page(s->grants_used[i]->pfn));
bvec_data = bvec_kmap_irq(bvec, &flags);
memcpy(bvec_data, shared_data + bvec->bv_offset,
bvec->bv_len);
bvec_kunmap_irq(bvec_data, &flags);
kunmap_atomic(shared_data);
offset += bvec->bv_len;
}
}
/* Add the persistent grant into the list of free grants */
for (i = 0; i < s->req.u.rw.nr_segments; i++) {
llist_add(&s->grants_used[i]->node, &info->persistent_gnts);
info->persistent_gnts_c++;
}
} }
static irqreturn_t blkif_interrupt(int irq, void *dev_id) static irqreturn_t blkif_interrupt(int irq, void *dev_id)
@ -783,7 +910,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
req = info->shadow[id].request; req = info->shadow[id].request;
if (bret->operation != BLKIF_OP_DISCARD) if (bret->operation != BLKIF_OP_DISCARD)
blkif_completion(&info->shadow[id]); blkif_completion(&info->shadow[id], info, bret);
if (add_id_to_freelist(info, id)) { if (add_id_to_freelist(info, id)) {
WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n", WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n",
@ -942,6 +1069,11 @@ again:
message = "writing protocol"; message = "writing protocol";
goto abort_transaction; goto abort_transaction;
} }
err = xenbus_printf(xbt, dev->nodename,
"feature-persistent", "%u", 1);
if (err)
dev_warn(&dev->dev,
"writing persistent grants feature to xenbus");
err = xenbus_transaction_end(xbt, 0); err = xenbus_transaction_end(xbt, 0);
if (err) { if (err) {
@ -1029,6 +1161,8 @@ static int blkfront_probe(struct xenbus_device *dev,
spin_lock_init(&info->io_lock); spin_lock_init(&info->io_lock);
info->xbdev = dev; info->xbdev = dev;
info->vdevice = vdevice; info->vdevice = vdevice;
init_llist_head(&info->persistent_gnts);
info->persistent_gnts_c = 0;
info->connected = BLKIF_STATE_DISCONNECTED; info->connected = BLKIF_STATE_DISCONNECTED;
INIT_WORK(&info->work, blkif_restart_queue); INIT_WORK(&info->work, blkif_restart_queue);
@ -1093,7 +1227,7 @@ static int blkif_recover(struct blkfront_info *info)
req->u.rw.seg[j].gref, req->u.rw.seg[j].gref,
info->xbdev->otherend_id, info->xbdev->otherend_id,
pfn_to_mfn(info->shadow[req->u.rw.id].frame[j]), pfn_to_mfn(info->shadow[req->u.rw.id].frame[j]),
rq_data_dir(info->shadow[req->u.rw.id].request)); 0);
} }
info->shadow[req->u.rw.id].req = *req; info->shadow[req->u.rw.id].req = *req;
@ -1225,7 +1359,7 @@ static void blkfront_connect(struct blkfront_info *info)
unsigned long sector_size; unsigned long sector_size;
unsigned int binfo; unsigned int binfo;
int err; int err;
int barrier, flush, discard; int barrier, flush, discard, persistent;
switch (info->connected) { switch (info->connected) {
case BLKIF_STATE_CONNECTED: case BLKIF_STATE_CONNECTED:
@ -1303,6 +1437,14 @@ static void blkfront_connect(struct blkfront_info *info)
if (!err && discard) if (!err && discard)
blkfront_setup_discard(info); blkfront_setup_discard(info);
err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
"feature-persistent", "%u", &persistent,
NULL);
if (err)
info->feature_persistent = 0;
else
info->feature_persistent = persistent;
err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size); err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size);
if (err) { if (err) {
xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s", xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",