bio: skip atomic inc/dec of ->bi_remaining for non-chains

[firefly-linux-kernel-4.4.55.git] / block / bio.c

diff --git a/block/bio.c b/block/bio.c
index f66a4eae16ee4a96c9469c7a9311de3437a923c5..117da319afb62c7a33d59479bf997a6245943440 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -270,7 +270,7 @@ void bio_init(struct bio *bio)
 {
        memset(bio, 0, sizeof(*bio));
        bio->bi_flags = 1 << BIO_UPTODATE;
-       atomic_set(&bio->bi_remaining, 1);
+       atomic_set(&bio->__bi_remaining, 1);
        atomic_set(&bio->bi_cnt, 1);
 }
 EXPORT_SYMBOL(bio_init);
@@ -292,8 +292,8 @@ void bio_reset(struct bio *bio)
        __bio_free(bio);
 
        memset(bio, 0, BIO_RESET_BYTES);
-       bio->bi_flags = flags|(1 << BIO_UPTODATE);
-       atomic_set(&bio->bi_remaining, 1);
+       bio->bi_flags = flags | (1 << BIO_UPTODATE);
+       atomic_set(&bio->__bi_remaining, 1);
 }
 EXPORT_SYMBOL(bio_reset);
 
@@ -320,7 +320,7 @@ void bio_chain(struct bio *bio, struct bio *parent)
 
        bio->bi_private = parent;
        bio->bi_end_io  = bio_chain_endio;
-       atomic_inc(&parent->bi_remaining);
+       bio_inc_remaining(parent);
 }
 EXPORT_SYMBOL(bio_chain);
 
@@ -1741,6 +1741,23 @@ void bio_flush_dcache_pages(struct bio *bi)
 EXPORT_SYMBOL(bio_flush_dcache_pages);
 #endif
 
+static inline bool bio_remaining_done(struct bio *bio)
+{
+       /*
+        * If we're not chaining, then ->__bi_remaining is always 1 and
+        * we always end io on the first invocation.
+        */
+       if (!bio_flagged(bio, BIO_CHAIN))
+               return true;
+
+       BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
+
+       if (atomic_dec_and_test(&bio->__bi_remaining))
+               return true;
+
+       return false;
+}
+
 /**
  * bio_endio - end I/O on a bio
  * @bio:       bio
@@ -1758,15 +1775,13 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
 void bio_endio(struct bio *bio, int error)
 {
        while (bio) {
-               BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
-
                if (error)
                        clear_bit(BIO_UPTODATE, &bio->bi_flags);
                else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
                        error = -EIO;
 
-               if (!atomic_dec_and_test(&bio->bi_remaining))
-                       return;
+               if (unlikely(!bio_remaining_done(bio)))
+                       break;
 
                /*
                 * Need to have a real endio function for chained bios,
@@ -1799,7 +1814,12 @@ EXPORT_SYMBOL(bio_endio);
  **/
 void bio_endio_nodec(struct bio *bio, int error)
 {
-       atomic_inc(&bio->bi_remaining);
+       /*
+        * If it's not flagged as a chain, we are not going to dec the count
+        */
+       if (bio_flagged(bio, BIO_CHAIN))
+               bio_inc_remaining(bio);
+
        bio_endio(bio, error);
 }
 EXPORT_SYMBOL(bio_endio_nodec);