lightnvm: pblk: clear flush point on completed writes
Move completion of syncs and clearing of flush points to the write completion path - this ensures that the data has been comitted to the media before completing bios containing syncs. Signed-off-by: Hans Holmberg <hans.holmberg@cnexlabs.com> Signed-off-by: Javier González <javier@cnexlabs.com> Signed-off-by: Matias Bjørling <m@bjorling.me> Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
8154d296d9
commit
533657c190
|
@ -353,17 +353,17 @@ static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
|
|||
unsigned int pos)
|
||||
{
|
||||
struct pblk_rb_entry *entry;
|
||||
unsigned int subm, flush_point;
|
||||
unsigned int sync, flush_point;
|
||||
|
||||
subm = READ_ONCE(rb->subm);
|
||||
sync = READ_ONCE(rb->sync);
|
||||
|
||||
if (pos == sync)
|
||||
return 0;
|
||||
|
||||
#ifdef CONFIG_NVM_DEBUG
|
||||
atomic_inc(&rb->inflight_flush_point);
|
||||
#endif
|
||||
|
||||
if (pos == subm)
|
||||
return 0;
|
||||
|
||||
flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
|
||||
entry = &rb->entries[flush_point];
|
||||
|
||||
|
@ -606,22 +606,6 @@ try:
|
|||
return NVM_IO_ERR;
|
||||
}
|
||||
|
||||
if (flags & PBLK_FLUSH_ENTRY) {
|
||||
unsigned int flush_point;
|
||||
|
||||
flush_point = READ_ONCE(rb->flush_point);
|
||||
if (flush_point == pos) {
|
||||
/* Protect flush points */
|
||||
smp_store_release(&rb->flush_point,
|
||||
EMPTY_ENTRY);
|
||||
}
|
||||
|
||||
flags &= ~PBLK_FLUSH_ENTRY;
|
||||
#ifdef CONFIG_NVM_DEBUG
|
||||
atomic_dec(&rb->inflight_flush_point);
|
||||
#endif
|
||||
}
|
||||
|
||||
flags &= ~PBLK_WRITTEN_DATA;
|
||||
flags |= PBLK_SUBMITTED_ENTRY;
|
||||
|
||||
|
@ -731,15 +715,24 @@ void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
|
|||
|
||||
unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
|
||||
{
|
||||
unsigned int sync;
|
||||
unsigned int i;
|
||||
|
||||
unsigned int sync, flush_point;
|
||||
lockdep_assert_held(&rb->s_lock);
|
||||
|
||||
sync = READ_ONCE(rb->sync);
|
||||
flush_point = READ_ONCE(rb->flush_point);
|
||||
|
||||
for (i = 0; i < nr_entries; i++)
|
||||
sync = (sync + 1) & (rb->nr_entries - 1);
|
||||
if (flush_point != EMPTY_ENTRY) {
|
||||
unsigned int secs_to_flush;
|
||||
|
||||
secs_to_flush = pblk_rb_ring_count(flush_point, sync,
|
||||
rb->nr_entries);
|
||||
if (secs_to_flush < nr_entries) {
|
||||
/* Protect flush points */
|
||||
smp_store_release(&rb->flush_point, EMPTY_ENTRY);
|
||||
}
|
||||
}
|
||||
|
||||
sync = (sync + nr_entries) & (rb->nr_entries - 1);
|
||||
|
||||
/* Protect from counts */
|
||||
smp_store_release(&rb->sync, sync);
|
||||
|
@ -747,22 +740,27 @@ unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
|
|||
return sync;
|
||||
}
|
||||
|
||||
/* Calculate how many sectors to submit up to the current flush point. */
|
||||
unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
|
||||
{
|
||||
unsigned int subm, flush_point;
|
||||
unsigned int count;
|
||||
unsigned int subm, sync, flush_point;
|
||||
unsigned int submitted, to_flush;
|
||||
|
||||
/* Protect flush points */
|
||||
flush_point = smp_load_acquire(&rb->flush_point);
|
||||
if (flush_point == EMPTY_ENTRY)
|
||||
return 0;
|
||||
|
||||
/* Protect syncs */
|
||||
sync = smp_load_acquire(&rb->sync);
|
||||
|
||||
subm = READ_ONCE(rb->subm);
|
||||
submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
|
||||
|
||||
/* The sync point itself counts as a sector to sync */
|
||||
count = pblk_rb_ring_count(flush_point, subm, rb->nr_entries) + 1;
|
||||
to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
|
||||
|
||||
return count;
|
||||
return (submitted < to_flush) ? (to_flush - submitted) : 0;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -21,13 +21,28 @@ static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,
|
|||
struct pblk_c_ctx *c_ctx)
|
||||
{
|
||||
struct bio *original_bio;
|
||||
struct pblk_rb *rwb = &pblk->rwb;
|
||||
unsigned long ret;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < c_ctx->nr_valid; i++) {
|
||||
struct pblk_w_ctx *w_ctx;
|
||||
int pos = c_ctx->sentry + i;
|
||||
int flags;
|
||||
|
||||
w_ctx = pblk_rb_w_ctx(rwb, pos);
|
||||
flags = READ_ONCE(w_ctx->flags);
|
||||
|
||||
if (flags & PBLK_FLUSH_ENTRY) {
|
||||
flags &= ~PBLK_FLUSH_ENTRY;
|
||||
/* Release flags on context. Protect from writes */
|
||||
smp_store_release(&w_ctx->flags, flags);
|
||||
|
||||
#ifdef CONFIG_NVM_DEBUG
|
||||
atomic_dec(&rwb->inflight_flush_point);
|
||||
#endif
|
||||
}
|
||||
|
||||
w_ctx = pblk_rb_w_ctx(&pblk->rwb, c_ctx->sentry + i);
|
||||
while ((original_bio = bio_list_pop(&w_ctx->bios)))
|
||||
bio_endio(original_bio);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue