From aad560b7f63b495f48a7232fd086c5913a676e6f Mon Sep 17 00:00:00 2001 From: Boaz Harrosh Date: Thu, 21 Nov 2013 17:58:08 +0200 Subject: [PATCH 1/5] ore: Fix wrong math in allocation of per device BIO At IO preparation we calculate the max pages at each device and allocate a BIO per device of that size. The calculation was wrong on some unaligned corner cases offset/length combination and would make prepare return with -ENOMEM. This would be bad for pnfs-objects that would in that case IO through MDS. And fatal for exofs were it would fail writes with EIO. Fix it by doing the proper math, that will work in all cases. (I ran a test with all possible offset/length combinations this time round). Also when reading we do not need to allocate for the parity units since we jump over them. Also lower the max_io_length to take into account the parity pages so not to allocate BIOs bigger than PAGE_SIZE CC: Stable Kernel Signed-off-by: Boaz Harrosh --- fs/exofs/ore.c | 37 +++++++++++++++++++++++++------------ include/scsi/osd_ore.h | 1 + 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/fs/exofs/ore.c b/fs/exofs/ore.c index b74422888604..85cde3e76290 100644 --- a/fs/exofs/ore.c +++ b/fs/exofs/ore.c @@ -103,7 +103,7 @@ int ore_verify_layout(unsigned total_comps, struct ore_layout *layout) layout->max_io_length = (BIO_MAX_PAGES_KMALLOC * PAGE_SIZE - layout->stripe_unit) * - layout->group_width; + (layout->group_width - layout->parity); if (layout->parity) { unsigned stripe_length = (layout->group_width - layout->parity) * @@ -286,7 +286,8 @@ int ore_get_rw_state(struct ore_layout *layout, struct ore_components *oc, if (length) { ore_calc_stripe_info(layout, offset, length, &ios->si); ios->length = ios->si.length; - ios->nr_pages = (ios->length + PAGE_SIZE - 1) / PAGE_SIZE; + ios->nr_pages = ((ios->offset & (PAGE_SIZE - 1)) + + ios->length + PAGE_SIZE - 1) / PAGE_SIZE; if (layout->parity) _ore_post_alloc_raid_stuff(ios); } @@ -536,6 +537,7 @@ void ore_calc_stripe_info(struct ore_layout *layout, u64 file_offset, u64 H = LmodS - G * T; u32 N = div_u64(H, U); + u32 Nlast; /* "H - (N * U)" is just "H % U" so it's bound to u32 */ u32 C = (u32)(H - (N * U)) / stripe_unit + G * group_width; @@ -568,6 +570,10 @@ void ore_calc_stripe_info(struct ore_layout *layout, u64 file_offset, si->length = T - H; if (si->length > length) si->length = length; + + Nlast = div_u64(H + si->length + U - 1, U); + si->maxdevUnits = Nlast - N; + si->M = M; } EXPORT_SYMBOL(ore_calc_stripe_info); @@ -583,13 +589,16 @@ int _ore_add_stripe_unit(struct ore_io_state *ios, unsigned *cur_pg, int ret; if (per_dev->bio == NULL) { - unsigned pages_in_stripe = ios->layout->group_width * - (ios->layout->stripe_unit / PAGE_SIZE); - unsigned nr_pages = ios->nr_pages * ios->layout->group_width / - (ios->layout->group_width - - ios->layout->parity); - unsigned bio_size = (nr_pages + pages_in_stripe) / - ios->layout->group_width; + unsigned bio_size; + + if (!ios->reading) { + bio_size = ios->si.maxdevUnits; + } else { + bio_size = (ios->si.maxdevUnits + 1) * + (ios->layout->group_width - ios->layout->parity) / + ios->layout->group_width; + } + bio_size *= (ios->layout->stripe_unit / PAGE_SIZE); per_dev->bio = bio_kmalloc(GFP_KERNEL, bio_size); if (unlikely(!per_dev->bio)) { @@ -609,8 +618,12 @@ int _ore_add_stripe_unit(struct ore_io_state *ios, unsigned *cur_pg, added_len = bio_add_pc_page(q, per_dev->bio, pages[pg], pglen, pgbase); if (unlikely(pglen != added_len)) { - ORE_DBGMSG("Failed bio_add_pc_page bi_vcnt=%u\n", - per_dev->bio->bi_vcnt); + /* If bi_vcnt == bi_max then this is a SW BUG */ + ORE_DBGMSG("Failed bio_add_pc_page bi_vcnt=0x%x " + "bi_max=0x%x BIO_MAX=0x%x cur_len=0x%x\n", + per_dev->bio->bi_vcnt, + per_dev->bio->bi_max_vecs, + BIO_MAX_PAGES_KMALLOC, cur_len); ret = -ENOMEM; goto out; } @@ -1098,7 +1111,7 @@ int ore_truncate(struct ore_layout *layout, struct ore_components *oc, size_attr->attr = g_attr_logical_length; size_attr->attr.val_ptr = &size_attr->newsize; - ORE_DBGMSG("trunc(0x%llx) obj_offset=0x%llx dev=%d\n", + ORE_DBGMSG2("trunc(0x%llx) obj_offset=0x%llx dev=%d\n", _LLU(oc->comps->obj.id), _LLU(obj_size), i); ret = _truncate_mirrors(ios, i * ios->layout->mirrors_p1, &size_attr->attr); diff --git a/include/scsi/osd_ore.h b/include/scsi/osd_ore.h index a5f9b960dfc8..6ca3265a4dca 100644 --- a/include/scsi/osd_ore.h +++ b/include/scsi/osd_ore.h @@ -102,6 +102,7 @@ struct ore_striping_info { unsigned unit_off; unsigned cur_pg; unsigned cur_comp; + unsigned maxdevUnits; }; struct ore_io_state; From 2deb76db092a91088494c00862bafc4fe7980adb Mon Sep 17 00:00:00 2001 From: Boaz Harrosh Date: Mon, 13 Jan 2014 19:10:40 +0200 Subject: [PATCH 2/5] ore: Don't crash on NULL bio in _clear_bio In the case of target returning OSD_ERR_PRI_CLEAR_PAGES when we only sent for attributes don't crash on NULL bio. This is an osd-target bug but don't crash regardless Signed-off-by: Boaz Harrosh --- fs/exofs/ore.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/exofs/ore.c b/fs/exofs/ore.c index 85cde3e76290..dae884694bd9 100644 --- a/fs/exofs/ore.c +++ b/fs/exofs/ore.c @@ -431,8 +431,12 @@ int ore_check_io(struct ore_io_state *ios, ore_on_dev_error on_dev_error) if (likely(!ret)) continue; - if (OSD_ERR_PRI_CLEAR_PAGES == osi.osd_err_pri) { - /* start read offset passed endof file */ + if ((OSD_ERR_PRI_CLEAR_PAGES == osi.osd_err_pri) && + per_dev->bio) { + /* start read offset passed endof file. + * Note: if we do not have bio it means read-attributes + * In this case we should return error to caller. + */ _clear_bio(per_dev->bio); ORE_DBGMSG("start read offset passed end of file " "offset=0x%llx, length=0x%llx\n", From d83c7eb65d9bf0a57e7d5ed87a5bd8e5ea6b1fb6 Mon Sep 17 00:00:00 2001 From: Boaz Harrosh Date: Mon, 13 Jan 2014 23:45:43 +0200 Subject: [PATCH 3/5] exofs: Allow O_DIRECT open With this minimal do nothing patch an application can open O_DIRECT and then actually do buffered sync IO instead. But the aio API is supported which is a good thing Signed-off-by: Boaz Harrosh --- fs/exofs/inode.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/exofs/inode.c b/fs/exofs/inode.c index a52a5d23c30b..7e7ba9ada405 100644 --- a/fs/exofs/inode.c +++ b/fs/exofs/inode.c @@ -961,6 +961,14 @@ static void exofs_invalidatepage(struct page *page, unsigned int offset, WARN_ON(1); } + + /* TODO: Should be easy enough to do proprly */ +static ssize_t exofs_direct_IO(int rw, struct kiocb *iocb, + const struct iovec *iov, loff_t offset, unsigned long nr_segs) +{ + return 0; +} + const struct address_space_operations exofs_aops = { .readpage = exofs_readpage, .readpages = exofs_readpages, @@ -974,7 +982,7 @@ const struct address_space_operations exofs_aops = { /* Not implemented Yet */ .bmap = NULL, /* TODO: use osd's OSD_ACT_READ_MAP */ - .direct_IO = NULL, /* TODO: Should be trivial to do */ + .direct_IO = exofs_direct_IO, /* With these NULL has special meaning or default is not exported */ .get_xip_mem = NULL, From c8592fcc66b2fe9cbd505f1faff810f8844d4a97 Mon Sep 17 00:00:00 2001 From: Boaz Harrosh Date: Tue, 14 Jan 2014 16:05:52 +0200 Subject: [PATCH 4/5] exofs: Allow corrupted directory entry to be empty file If there was an error in fetching an object or extracting inode info from attributes. Which means corrupted storage. Let it be an empty ZERO dated directory entry so it can be deleted. Otherwise the all directory will be inaccessible. This does not loose data, because if there is an orphan object somewhere it will be recovered by fschk. But usually this only means corrupted dir entry. The object was never generated and only its link exist. This way we can delete the bad entry. Signed-off-by: Boaz Harrosh --- fs/exofs/inode.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/exofs/inode.c b/fs/exofs/inode.c index 7e7ba9ada405..390224a162d0 100644 --- a/fs/exofs/inode.c +++ b/fs/exofs/inode.c @@ -1102,8 +1102,7 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi, /* If object is lost on target we might as well enable it's * delete. */ - if ((ret == -ENOENT) || (ret == -EINVAL)) - ret = 0; + ret = 0; goto out; } From 19350e7627a6f3b0f662cbd2eb1128c9961a41fe Mon Sep 17 00:00:00 2001 From: Boaz Harrosh Date: Tue, 14 Jan 2014 16:26:13 +0200 Subject: [PATCH 5/5] exofs: Print less in r4w In debug mode exofs is too verbose. Hiding the real problems remove some trivial stuff. Also fix some other prints. Signed-off-by: Boaz Harrosh --- fs/exofs/inode.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/exofs/inode.c b/fs/exofs/inode.c index 390224a162d0..ee4317faccb1 100644 --- a/fs/exofs/inode.c +++ b/fs/exofs/inode.c @@ -577,7 +577,7 @@ static struct page *__r4w_get_page(void *priv, u64 offset, bool *uptodate) if (offset >= i_size) { *uptodate = true; - EXOFS_DBGMSG("offset >= i_size index=0x%lx\n", index); + EXOFS_DBGMSG2("offset >= i_size index=0x%lx\n", index); return ZERO_PAGE(0); } @@ -596,10 +596,10 @@ static struct page *__r4w_get_page(void *priv, u64 offset, bool *uptodate) *uptodate = true; else *uptodate = PageUptodate(page); - EXOFS_DBGMSG("index=0x%lx uptodate=%d\n", index, *uptodate); + EXOFS_DBGMSG2("index=0x%lx uptodate=%d\n", index, *uptodate); return page; } else { - EXOFS_DBGMSG("YES that_locked_page index=0x%lx\n", + EXOFS_DBGMSG2("YES that_locked_page index=0x%lx\n", pcol->that_locked_page->index); *uptodate = true; return pcol->that_locked_page; @@ -611,11 +611,11 @@ static void __r4w_put_page(void *priv, struct page *page) struct page_collect *pcol = priv; if ((pcol->that_locked_page != page) && (ZERO_PAGE(0) != page)) { - EXOFS_DBGMSG("index=0x%lx\n", page->index); + EXOFS_DBGMSG2("index=0x%lx\n", page->index); page_cache_release(page); return; } - EXOFS_DBGMSG("that_locked_page index=0x%lx\n", + EXOFS_DBGMSG2("that_locked_page index=0x%lx\n", ZERO_PAGE(0) == page ? -1 : page->index); } @@ -1018,7 +1018,7 @@ static int _do_truncate(struct inode *inode, loff_t newsize) if (likely(!ret)) truncate_setsize(inode, newsize); - EXOFS_DBGMSG("(0x%lx) size=0x%llx ret=>%d\n", + EXOFS_DBGMSG2("(0x%lx) size=0x%llx ret=>%d\n", inode->i_ino, newsize, ret); return ret; } @@ -1108,7 +1108,7 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi, ret = extract_attr_from_ios(ios, &attrs[0]); if (ret) { - EXOFS_ERR("%s: extract_attr of inode_data failed\n", __func__); + EXOFS_ERR("%s: extract_attr 0 of inode failed\n", __func__); goto out; } WARN_ON(attrs[0].len != EXOFS_INO_ATTR_SIZE); @@ -1116,7 +1116,7 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi, ret = extract_attr_from_ios(ios, &attrs[1]); if (ret) { - EXOFS_ERR("%s: extract_attr of inode_data failed\n", __func__); + EXOFS_ERR("%s: extract_attr 1 of inode failed\n", __func__); goto out; } if (attrs[1].len) { @@ -1131,7 +1131,7 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi, ret = extract_attr_from_ios(ios, &attrs[2]); if (ret) { - EXOFS_ERR("%s: extract_attr of inode_data failed\n", __func__); + EXOFS_ERR("%s: extract_attr 2 of inode failed\n", __func__); goto out; } if (attrs[2].len) {