mmc: block: replace __blk_end_request() with blk_end_request()

[rk emmc: read rate up to 78MB/s, write rate up to 26MB/s]

For completing any block request, MMC block driver is calling:
	spin_lock_irq(queue)
	__blk_end_request()
	spin_unlock_irq(queue)

But if we analyze the sources of latency in kernel using ftrace,
__blk_end_request() function at times may take up to 6.5ms with
spinlock held and irq disabled.

__blk_end_request() calls couple of functions and ftrace output
shows that blk_update_bidi_request() function is almost taking 6ms.
There are 2 function to end the current request: ___blk_end_request()
and blk_end_request(). Both these functions do same thing except
that blk_end_request() function doesn't take up the spinlock
while calling the blk_update_bidi_request().

This patch replaces all __blk_end_request() calls with
blk_end_request() and __blk_end_request_all() calls with
blk_end_request_all().

Testing done: 20 process concurrent read/write on sd card
and eMMC. Ran this test for almost a day on multicore system
and no errors observed.

This change is not meant for improving MMC throughput; it's basically
about becoming fair to other threads/interrupts in the system. By
holding spin lock and interrupts disabled for longer duration, we
won't allow other threads/interrupts to run at all.  Actually slight
performance degradation at file system level can be expected as we
are not holding the spin lock during blk_update_bidi_request() which
means our mmcqd thread may get preempted for other high priority
thread or any interrupt in the system.

These are performance numbers (100MB file write) with eMMC running
in DDR mode:

Without this patch:
	Name of the Test   Value   Unit
	LMDD Read Test     53.79   MBPS
	LMDD Write Test    18.86   MBPS
	IOZONE  Read Test  51.65   MBPS
	IOZONE  Write Test 24.36   MBPS

With this patch:
	Name of the Test    Value  Unit
	LMDD Read Test      52.94  MBPS
	LMDD Write Test     16.70  MBPS
	IOZONE  Read Test   52.08  MBPS
	IOZONE  Write Test  23.29  MBPS

Read numbers are fine. Write numbers are bit down (especially LMDD
write), may be because write requests normally have large transfer
size and which means there are chances that while mmcq is executing
blk_update_bidi_request(), it may get interrupted by interrupts or
other high priority thread.

Signed-off-by: Subhash Jadavani <subhashj@codeaurora.org>
Reviewed-by: Namjae Jeon <linkinjeon@gmail.com>
Signed-off-by: Chris Ball <cjb@laptop.org>
Conflicts:
	drivers/mmc/card/block.c

diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c
index ff810129feb27184a59c982be8e9308731289247..462098047bbf9f4b3ae4064a200c0b83b3177759 100644 (file)
--- a/drivers/mmc/card/block.c
+++ b/drivers/mmc/card/block.c
@@ -693,8 +693,7 @@ static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
        }
        return ERR_CONTINUE;
 }
-
-static int mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
+static int sdmmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
 {
        struct mmc_blk_data *md = mq->data;
        struct mmc_card *card = md->queue.card;
@@ -733,7 +732,54 @@ out:
        return err ? 0 : 1;
 }
 
-static int mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
+static int emmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
+{
+       struct mmc_blk_data *md = mq->data;
+       struct mmc_card *card = md->queue.card;
+       unsigned int from, nr, arg;
+       int err = 0;
+
+       if (!mmc_can_erase(card)) {
+               err = -EOPNOTSUPP;
+               goto out;
+       }
+
+       from = blk_rq_pos(req);
+       nr = blk_rq_sectors(req);
+
+       if (mmc_can_trim(card))
+               arg = MMC_TRIM_ARG;
+       else
+               arg = MMC_ERASE_ARG;
+
+       if (card->quirks & MMC_QUIRK_INAND_CMD38) {
+               err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
+                                INAND_CMD38_ARG_EXT_CSD,
+                                arg == MMC_TRIM_ARG ?
+                                INAND_CMD38_ARG_TRIM :
+                                INAND_CMD38_ARG_ERASE,
+                                0);
+               if (err)
+                       goto out;
+       }
+       err = mmc_erase(card, from, nr, arg);
+out:
+       blk_end_request(req, err, blk_rq_bytes(req));
+
+       return err ? 0 : 1;
+}
+
+static int mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
+{
+       struct mmc_blk_data *md = mq->data;
+       struct mmc_card *card = md->queue.card;
+
+       if(HOST_IS_EMMC(card->host))
+               return emmc_blk_issue_discard_rq(mq, req);
+       else
+               return sdmmc_blk_issue_discard_rq(mq, req);
+}
+static int sdmmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
                                       struct request *req)
 {
        struct mmc_blk_data *md = mq->data;
@@ -783,8 +829,68 @@ out:
 
        return err ? 0 : 1;
 }
+static int emmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
+                                      struct request *req)
+{
+       struct mmc_blk_data *md = mq->data;
+       struct mmc_card *card = md->queue.card;
+       unsigned int from, nr, arg;
+       int err = 0;
 
-static int mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
+       if (!mmc_can_secure_erase_trim(card)) {
+               err = -EOPNOTSUPP;
+               goto out;
+       }
+
+       from = blk_rq_pos(req);
+       nr = blk_rq_sectors(req);
+
+       if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
+               arg = MMC_SECURE_TRIM1_ARG;
+       else
+               arg = MMC_SECURE_ERASE_ARG;
+
+       if (card->quirks & MMC_QUIRK_INAND_CMD38) {
+               err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
+                                INAND_CMD38_ARG_EXT_CSD,
+                                arg == MMC_SECURE_TRIM1_ARG ?
+                                INAND_CMD38_ARG_SECTRIM1 :
+                                INAND_CMD38_ARG_SECERASE,
+                                0);
+               if (err)
+                       goto out;
+       }
+       err = mmc_erase(card, from, nr, arg);
+       if (!err && arg == MMC_SECURE_TRIM1_ARG) {
+               if (card->quirks & MMC_QUIRK_INAND_CMD38) {
+                       err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
+                                        INAND_CMD38_ARG_EXT_CSD,
+                                        INAND_CMD38_ARG_SECTRIM2,
+                                        0);
+                       if (err)
+                               goto out;
+               }
+               err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG);
+       }
+out:
+       blk_end_request(req, err, blk_rq_bytes(req));
+
+       return err ? 0 : 1;
+}
+
+static int mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
+                                      struct request *req)
+{
+       struct mmc_blk_data *md = mq->data;
+       struct mmc_card *card = md->queue.card;
+       
+       if(HOST_IS_EMMC(card->host))
+               return emmc_blk_issue_secdiscard_rq(mq, req);
+       else
+               return sdmmc_blk_issue_secdiscard_rq(mq, req);
+}
+
+static int sdmmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
 {
        struct mmc_blk_data *md = mq->data;
 
@@ -799,6 +905,30 @@ static int mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
        return 1;
 }
 
+
+static int emmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
+{
+       //struct mmc_blk_data *md = mq->data;
+
+       /*
+        * No-op, only service this because we need REQ_FUA for reliable
+        * writes.
+        */
+       blk_end_request_all(req, 0);
+
+       return 1;
+}
+
+static int mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
+{
+       struct mmc_blk_data *md = mq->data;
+       struct mmc_card *card = md->queue.card;
+
+       if(HOST_IS_EMMC(card->host))
+               return emmc_blk_issue_flush(mq, req);
+       else
+               return sdmmc_blk_issue_flush(mq, req);
+}
 /*
  * Reformat current write as a reliable write, supporting
  * both legacy and the enhanced reliable write MMC cards.
@@ -1369,10 +1499,8 @@ static int emmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
                        /*
                         * A block was successfully transferred.
                         */
-                       spin_lock_irq(&md->lock);
-                       ret = __blk_end_request(req, 0,
+                       ret = blk_end_request(req, 0,
                                                brq->data.bytes_xfered);
-                       spin_unlock_irq(&md->lock);
                        if (status == MMC_BLK_SUCCESS && ret) {
                                /*
                                 * The blk_end_request has returned non zero
@@ -1403,10 +1531,8 @@ static int emmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
                         * time, so we only reach here after trying to
                         * read a single sector.
                         */
-                       spin_lock_irq(&md->lock);
-                       ret = __blk_end_request(req, -EIO,
+                       ret = blk_end_request(req, -EIO,
                                                brq->data.blksz);
-                       spin_unlock_irq(&md->lock);
                        if (!ret)
                                goto start_new_req;
                        break;
@@ -1438,21 +1564,15 @@ static int emmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 
                blocks = mmc_sd_num_wr_blocks(card);
                if (blocks != (u32)-1) {
-                       spin_lock_irq(&md->lock);
-                       ret = __blk_end_request(req, 0, blocks << 9);
-                       spin_unlock_irq(&md->lock);
+                       ret = blk_end_request(req, 0, blocks << 9);
                }
        } else {
-               spin_lock_irq(&md->lock);
-               ret = __blk_end_request(req, 0, brq->data.bytes_xfered);
-               spin_unlock_irq(&md->lock);
+               ret = blk_end_request(req, 0, brq->data.bytes_xfered);
        }
 
  cmd_abort:
-       spin_lock_irq(&md->lock);
        while (ret)
-               ret = __blk_end_request(req, -EIO, blk_rq_cur_bytes(req));
-       spin_unlock_irq(&md->lock);
+               ret = blk_end_request(req, -EIO, blk_rq_cur_bytes(req));
 
  start_new_req:
        if (rqc) {