fcdc585278bf0ee71b750976d784578dde0b32c2
[firefly-linux-kernel-4.4.55.git] / drivers / scsi / scsi_lib.c
1 /*
2  * Copyright (C) 1999 Eric Youngdale
3  * Copyright (C) 2014 Christoph Hellwig
4  *
5  *  SCSI queueing library.
6  *      Initial versions: Eric Youngdale (eric@andante.org).
7  *                        Based upon conversations with large numbers
8  *                        of people at Linux Expo.
9  */
10
11 #include <linux/bio.h>
12 #include <linux/bitops.h>
13 #include <linux/blkdev.h>
14 #include <linux/completion.h>
15 #include <linux/kernel.h>
16 #include <linux/export.h>
17 #include <linux/mempool.h>
18 #include <linux/slab.h>
19 #include <linux/init.h>
20 #include <linux/pci.h>
21 #include <linux/delay.h>
22 #include <linux/hardirq.h>
23 #include <linux/scatterlist.h>
24 #include <linux/blk-mq.h>
25 #include <linux/ratelimit.h>
26
27 #include <scsi/scsi.h>
28 #include <scsi/scsi_cmnd.h>
29 #include <scsi/scsi_dbg.h>
30 #include <scsi/scsi_device.h>
31 #include <scsi/scsi_driver.h>
32 #include <scsi/scsi_eh.h>
33 #include <scsi/scsi_host.h>
34
35 #include <trace/events/scsi.h>
36
37 #include "scsi_priv.h"
38 #include "scsi_logging.h"
39
40
41 #define SG_MEMPOOL_NR           ARRAY_SIZE(scsi_sg_pools)
42 #define SG_MEMPOOL_SIZE         2
43
44 struct scsi_host_sg_pool {
45         size_t          size;
46         char            *name;
47         struct kmem_cache       *slab;
48         mempool_t       *pool;
49 };
50
51 #define SP(x) { .size = x, "sgpool-" __stringify(x) }
52 #if (SCSI_MAX_SG_SEGMENTS < 32)
53 #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
54 #endif
55 static struct scsi_host_sg_pool scsi_sg_pools[] = {
56         SP(8),
57         SP(16),
58 #if (SCSI_MAX_SG_SEGMENTS > 32)
59         SP(32),
60 #if (SCSI_MAX_SG_SEGMENTS > 64)
61         SP(64),
62 #if (SCSI_MAX_SG_SEGMENTS > 128)
63         SP(128),
64 #if (SCSI_MAX_SG_SEGMENTS > 256)
65 #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
66 #endif
67 #endif
68 #endif
69 #endif
70         SP(SCSI_MAX_SG_SEGMENTS)
71 };
72 #undef SP
73
74 struct kmem_cache *scsi_sdb_cache;
75
76 /*
77  * When to reinvoke queueing after a resource shortage. It's 3 msecs to
78  * not change behaviour from the previous unplug mechanism, experimentation
79  * may prove this needs changing.
80  */
81 #define SCSI_QUEUE_DELAY        3
82
83 static void
84 scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
85 {
86         struct Scsi_Host *host = cmd->device->host;
87         struct scsi_device *device = cmd->device;
88         struct scsi_target *starget = scsi_target(device);
89
90         /*
91          * Set the appropriate busy bit for the device/host.
92          *
93          * If the host/device isn't busy, assume that something actually
94          * completed, and that we should be able to queue a command now.
95          *
96          * Note that the prior mid-layer assumption that any host could
97          * always queue at least one command is now broken.  The mid-layer
98          * will implement a user specifiable stall (see
99          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
100          * if a command is requeued with no other commands outstanding
101          * either for the device or for the host.
102          */
103         switch (reason) {
104         case SCSI_MLQUEUE_HOST_BUSY:
105                 atomic_set(&host->host_blocked, host->max_host_blocked);
106                 break;
107         case SCSI_MLQUEUE_DEVICE_BUSY:
108         case SCSI_MLQUEUE_EH_RETRY:
109                 atomic_set(&device->device_blocked,
110                            device->max_device_blocked);
111                 break;
112         case SCSI_MLQUEUE_TARGET_BUSY:
113                 atomic_set(&starget->target_blocked,
114                            starget->max_target_blocked);
115                 break;
116         }
117 }
118
119 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
120 {
121         struct scsi_device *sdev = cmd->device;
122         struct request_queue *q = cmd->request->q;
123
124         blk_mq_requeue_request(cmd->request);
125         blk_mq_kick_requeue_list(q);
126         put_device(&sdev->sdev_gendev);
127 }
128
129 /**
130  * __scsi_queue_insert - private queue insertion
131  * @cmd: The SCSI command being requeued
132  * @reason:  The reason for the requeue
133  * @unbusy: Whether the queue should be unbusied
134  *
135  * This is a private queue insertion.  The public interface
136  * scsi_queue_insert() always assumes the queue should be unbusied
137  * because it's always called before the completion.  This function is
138  * for a requeue after completion, which should only occur in this
139  * file.
140  */
141 static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
142 {
143         struct scsi_device *device = cmd->device;
144         struct request_queue *q = device->request_queue;
145         unsigned long flags;
146
147         SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
148                 "Inserting command %p into mlqueue\n", cmd));
149
150         scsi_set_blocked(cmd, reason);
151
152         /*
153          * Decrement the counters, since these commands are no longer
154          * active on the host/device.
155          */
156         if (unbusy)
157                 scsi_device_unbusy(device);
158
159         /*
160          * Requeue this command.  It will go before all other commands
161          * that are already in the queue. Schedule requeue work under
162          * lock such that the kblockd_schedule_work() call happens
163          * before blk_cleanup_queue() finishes.
164          */
165         cmd->result = 0;
166         if (q->mq_ops) {
167                 scsi_mq_requeue_cmd(cmd);
168                 return;
169         }
170         spin_lock_irqsave(q->queue_lock, flags);
171         blk_requeue_request(q, cmd->request);
172         kblockd_schedule_work(&device->requeue_work);
173         spin_unlock_irqrestore(q->queue_lock, flags);
174 }
175
176 /*
177  * Function:    scsi_queue_insert()
178  *
179  * Purpose:     Insert a command in the midlevel queue.
180  *
181  * Arguments:   cmd    - command that we are adding to queue.
182  *              reason - why we are inserting command to queue.
183  *
184  * Lock status: Assumed that lock is not held upon entry.
185  *
186  * Returns:     Nothing.
187  *
188  * Notes:       We do this for one of two cases.  Either the host is busy
189  *              and it cannot accept any more commands for the time being,
190  *              or the device returned QUEUE_FULL and can accept no more
191  *              commands.
192  * Notes:       This could be called either from an interrupt context or a
193  *              normal process context.
194  */
195 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
196 {
197         __scsi_queue_insert(cmd, reason, 1);
198 }
199 /**
200  * scsi_execute - insert request and wait for the result
201  * @sdev:       scsi device
202  * @cmd:        scsi command
203  * @data_direction: data direction
204  * @buffer:     data buffer
205  * @bufflen:    len of buffer
206  * @sense:      optional sense buffer
207  * @timeout:    request timeout in seconds
208  * @retries:    number of times to retry request
209  * @flags:      or into request flags;
210  * @resid:      optional residual length
211  *
212  * returns the req->errors value which is the scsi_cmnd result
213  * field.
214  */
215 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
216                  int data_direction, void *buffer, unsigned bufflen,
217                  unsigned char *sense, int timeout, int retries, u64 flags,
218                  int *resid)
219 {
220         struct request *req;
221         int write = (data_direction == DMA_TO_DEVICE);
222         int ret = DRIVER_ERROR << 24;
223
224         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
225         if (IS_ERR(req))
226                 return ret;
227         blk_rq_set_block_pc(req);
228
229         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,
230                                         buffer, bufflen, __GFP_WAIT))
231                 goto out;
232
233         req->cmd_len = COMMAND_SIZE(cmd[0]);
234         memcpy(req->cmd, cmd, req->cmd_len);
235         req->sense = sense;
236         req->sense_len = 0;
237         req->retries = retries;
238         req->timeout = timeout;
239         req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
240
241         /*
242          * head injection *required* here otherwise quiesce won't work
243          */
244         blk_execute_rq(req->q, NULL, req, 1);
245
246         /*
247          * Some devices (USB mass-storage in particular) may transfer
248          * garbage data together with a residue indicating that the data
249          * is invalid.  Prevent the garbage from being misinterpreted
250          * and prevent security leaks by zeroing out the excess data.
251          */
252         if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
253                 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
254
255         if (resid)
256                 *resid = req->resid_len;
257         ret = req->errors;
258  out:
259         blk_put_request(req);
260
261         return ret;
262 }
263 EXPORT_SYMBOL(scsi_execute);
264
265 int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd,
266                      int data_direction, void *buffer, unsigned bufflen,
267                      struct scsi_sense_hdr *sshdr, int timeout, int retries,
268                      int *resid, u64 flags)
269 {
270         char *sense = NULL;
271         int result;
272         
273         if (sshdr) {
274                 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
275                 if (!sense)
276                         return DRIVER_ERROR << 24;
277         }
278         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
279                               sense, timeout, retries, flags, resid);
280         if (sshdr)
281                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
282
283         kfree(sense);
284         return result;
285 }
286 EXPORT_SYMBOL(scsi_execute_req_flags);
287
288 /*
289  * Function:    scsi_init_cmd_errh()
290  *
291  * Purpose:     Initialize cmd fields related to error handling.
292  *
293  * Arguments:   cmd     - command that is ready to be queued.
294  *
295  * Notes:       This function has the job of initializing a number of
296  *              fields related to error handling.   Typically this will
297  *              be called once for each command, as required.
298  */
299 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
300 {
301         cmd->serial_number = 0;
302         scsi_set_resid(cmd, 0);
303         memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
304         if (cmd->cmd_len == 0)
305                 cmd->cmd_len = scsi_command_size(cmd->cmnd);
306 }
307
308 void scsi_device_unbusy(struct scsi_device *sdev)
309 {
310         struct Scsi_Host *shost = sdev->host;
311         struct scsi_target *starget = scsi_target(sdev);
312         unsigned long flags;
313
314         atomic_dec(&shost->host_busy);
315         if (starget->can_queue > 0)
316                 atomic_dec(&starget->target_busy);
317
318         if (unlikely(scsi_host_in_recovery(shost) &&
319                      (shost->host_failed || shost->host_eh_scheduled))) {
320                 spin_lock_irqsave(shost->host_lock, flags);
321                 scsi_eh_wakeup(shost);
322                 spin_unlock_irqrestore(shost->host_lock, flags);
323         }
324
325         atomic_dec(&sdev->device_busy);
326 }
327
328 static void scsi_kick_queue(struct request_queue *q)
329 {
330         if (q->mq_ops)
331                 blk_mq_start_hw_queues(q);
332         else
333                 blk_run_queue(q);
334 }
335
336 /*
337  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
338  * and call blk_run_queue for all the scsi_devices on the target -
339  * including current_sdev first.
340  *
341  * Called with *no* scsi locks held.
342  */
343 static void scsi_single_lun_run(struct scsi_device *current_sdev)
344 {
345         struct Scsi_Host *shost = current_sdev->host;
346         struct scsi_device *sdev, *tmp;
347         struct scsi_target *starget = scsi_target(current_sdev);
348         unsigned long flags;
349
350         spin_lock_irqsave(shost->host_lock, flags);
351         starget->starget_sdev_user = NULL;
352         spin_unlock_irqrestore(shost->host_lock, flags);
353
354         /*
355          * Call blk_run_queue for all LUNs on the target, starting with
356          * current_sdev. We race with others (to set starget_sdev_user),
357          * but in most cases, we will be first. Ideally, each LU on the
358          * target would get some limited time or requests on the target.
359          */
360         scsi_kick_queue(current_sdev->request_queue);
361
362         spin_lock_irqsave(shost->host_lock, flags);
363         if (starget->starget_sdev_user)
364                 goto out;
365         list_for_each_entry_safe(sdev, tmp, &starget->devices,
366                         same_target_siblings) {
367                 if (sdev == current_sdev)
368                         continue;
369                 if (scsi_device_get(sdev))
370                         continue;
371
372                 spin_unlock_irqrestore(shost->host_lock, flags);
373                 scsi_kick_queue(sdev->request_queue);
374                 spin_lock_irqsave(shost->host_lock, flags);
375         
376                 scsi_device_put(sdev);
377         }
378  out:
379         spin_unlock_irqrestore(shost->host_lock, flags);
380 }
381
382 static inline bool scsi_device_is_busy(struct scsi_device *sdev)
383 {
384         if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
385                 return true;
386         if (atomic_read(&sdev->device_blocked) > 0)
387                 return true;
388         return false;
389 }
390
391 static inline bool scsi_target_is_busy(struct scsi_target *starget)
392 {
393         if (starget->can_queue > 0) {
394                 if (atomic_read(&starget->target_busy) >= starget->can_queue)
395                         return true;
396                 if (atomic_read(&starget->target_blocked) > 0)
397                         return true;
398         }
399         return false;
400 }
401
402 static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
403 {
404         if (shost->can_queue > 0 &&
405             atomic_read(&shost->host_busy) >= shost->can_queue)
406                 return true;
407         if (atomic_read(&shost->host_blocked) > 0)
408                 return true;
409         if (shost->host_self_blocked)
410                 return true;
411         return false;
412 }
413
414 static void scsi_starved_list_run(struct Scsi_Host *shost)
415 {
416         LIST_HEAD(starved_list);
417         struct scsi_device *sdev;
418         unsigned long flags;
419
420         spin_lock_irqsave(shost->host_lock, flags);
421         list_splice_init(&shost->starved_list, &starved_list);
422
423         while (!list_empty(&starved_list)) {
424                 struct request_queue *slq;
425
426                 /*
427                  * As long as shost is accepting commands and we have
428                  * starved queues, call blk_run_queue. scsi_request_fn
429                  * drops the queue_lock and can add us back to the
430                  * starved_list.
431                  *
432                  * host_lock protects the starved_list and starved_entry.
433                  * scsi_request_fn must get the host_lock before checking
434                  * or modifying starved_list or starved_entry.
435                  */
436                 if (scsi_host_is_busy(shost))
437                         break;
438
439                 sdev = list_entry(starved_list.next,
440                                   struct scsi_device, starved_entry);
441                 list_del_init(&sdev->starved_entry);
442                 if (scsi_target_is_busy(scsi_target(sdev))) {
443                         list_move_tail(&sdev->starved_entry,
444                                        &shost->starved_list);
445                         continue;
446                 }
447
448                 /*
449                  * Once we drop the host lock, a racing scsi_remove_device()
450                  * call may remove the sdev from the starved list and destroy
451                  * it and the queue.  Mitigate by taking a reference to the
452                  * queue and never touching the sdev again after we drop the
453                  * host lock.  Note: if __scsi_remove_device() invokes
454                  * blk_cleanup_queue() before the queue is run from this
455                  * function then blk_run_queue() will return immediately since
456                  * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING.
457                  */
458                 slq = sdev->request_queue;
459                 if (!blk_get_queue(slq))
460                         continue;
461                 spin_unlock_irqrestore(shost->host_lock, flags);
462
463                 scsi_kick_queue(slq);
464                 blk_put_queue(slq);
465
466                 spin_lock_irqsave(shost->host_lock, flags);
467         }
468         /* put any unprocessed entries back */
469         list_splice(&starved_list, &shost->starved_list);
470         spin_unlock_irqrestore(shost->host_lock, flags);
471 }
472
473 /*
474  * Function:   scsi_run_queue()
475  *
476  * Purpose:    Select a proper request queue to serve next
477  *
478  * Arguments:  q       - last request's queue
479  *
480  * Returns:     Nothing
481  *
482  * Notes:      The previous command was completely finished, start
483  *             a new one if possible.
484  */
485 static void scsi_run_queue(struct request_queue *q)
486 {
487         struct scsi_device *sdev = q->queuedata;
488
489         if (scsi_target(sdev)->single_lun)
490                 scsi_single_lun_run(sdev);
491         if (!list_empty(&sdev->host->starved_list))
492                 scsi_starved_list_run(sdev->host);
493
494         if (q->mq_ops)
495                 blk_mq_start_stopped_hw_queues(q, false);
496         else
497                 blk_run_queue(q);
498 }
499
500 void scsi_requeue_run_queue(struct work_struct *work)
501 {
502         struct scsi_device *sdev;
503         struct request_queue *q;
504
505         sdev = container_of(work, struct scsi_device, requeue_work);
506         q = sdev->request_queue;
507         scsi_run_queue(q);
508 }
509
510 /*
511  * Function:    scsi_requeue_command()
512  *
513  * Purpose:     Handle post-processing of completed commands.
514  *
515  * Arguments:   q       - queue to operate on
516  *              cmd     - command that may need to be requeued.
517  *
518  * Returns:     Nothing
519  *
520  * Notes:       After command completion, there may be blocks left
521  *              over which weren't finished by the previous command
522  *              this can be for a number of reasons - the main one is
523  *              I/O errors in the middle of the request, in which case
524  *              we need to request the blocks that come after the bad
525  *              sector.
526  * Notes:       Upon return, cmd is a stale pointer.
527  */
528 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
529 {
530         struct scsi_device *sdev = cmd->device;
531         struct request *req = cmd->request;
532         unsigned long flags;
533
534         spin_lock_irqsave(q->queue_lock, flags);
535         blk_unprep_request(req);
536         req->special = NULL;
537         scsi_put_command(cmd);
538         blk_requeue_request(q, req);
539         spin_unlock_irqrestore(q->queue_lock, flags);
540
541         scsi_run_queue(q);
542
543         put_device(&sdev->sdev_gendev);
544 }
545
546 void scsi_run_host_queues(struct Scsi_Host *shost)
547 {
548         struct scsi_device *sdev;
549
550         shost_for_each_device(sdev, shost)
551                 scsi_run_queue(sdev->request_queue);
552 }
553
554 static inline unsigned int scsi_sgtable_index(unsigned short nents)
555 {
556         unsigned int index;
557
558         BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
559
560         if (nents <= 8)
561                 index = 0;
562         else
563                 index = get_count_order(nents) - 3;
564
565         return index;
566 }
567
568 static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
569 {
570         struct scsi_host_sg_pool *sgp;
571
572         sgp = scsi_sg_pools + scsi_sgtable_index(nents);
573         mempool_free(sgl, sgp->pool);
574 }
575
576 static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
577 {
578         struct scsi_host_sg_pool *sgp;
579
580         sgp = scsi_sg_pools + scsi_sgtable_index(nents);
581         return mempool_alloc(sgp->pool, gfp_mask);
582 }
583
584 static void scsi_free_sgtable(struct scsi_data_buffer *sdb, bool mq)
585 {
586         if (mq && sdb->table.nents <= SCSI_MAX_SG_SEGMENTS)
587                 return;
588         __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, mq, scsi_sg_free);
589 }
590
591 static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, bool mq)
592 {
593         struct scatterlist *first_chunk = NULL;
594         gfp_t gfp_mask = mq ? GFP_NOIO : GFP_ATOMIC;
595         int ret;
596
597         BUG_ON(!nents);
598
599         if (mq) {
600                 if (nents <= SCSI_MAX_SG_SEGMENTS) {
601                         sdb->table.nents = nents;
602                         sg_init_table(sdb->table.sgl, sdb->table.nents);
603                         return 0;
604                 }
605                 first_chunk = sdb->table.sgl;
606         }
607
608         ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
609                                first_chunk, gfp_mask, scsi_sg_alloc);
610         if (unlikely(ret))
611                 scsi_free_sgtable(sdb, mq);
612         return ret;
613 }
614
615 static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
616 {
617         if (cmd->request->cmd_type == REQ_TYPE_FS) {
618                 struct scsi_driver *drv = scsi_cmd_to_driver(cmd);
619
620                 if (drv->uninit_command)
621                         drv->uninit_command(cmd);
622         }
623 }
624
625 static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
626 {
627         if (cmd->sdb.table.nents)
628                 scsi_free_sgtable(&cmd->sdb, true);
629         if (cmd->request->next_rq && cmd->request->next_rq->special)
630                 scsi_free_sgtable(cmd->request->next_rq->special, true);
631         if (scsi_prot_sg_count(cmd))
632                 scsi_free_sgtable(cmd->prot_sdb, true);
633 }
634
635 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
636 {
637         struct scsi_device *sdev = cmd->device;
638         struct Scsi_Host *shost = sdev->host;
639         unsigned long flags;
640
641         scsi_mq_free_sgtables(cmd);
642         scsi_uninit_cmd(cmd);
643
644         if (shost->use_cmd_list) {
645                 BUG_ON(list_empty(&cmd->list));
646                 spin_lock_irqsave(&sdev->list_lock, flags);
647                 list_del_init(&cmd->list);
648                 spin_unlock_irqrestore(&sdev->list_lock, flags);
649         }
650 }
651
652 /*
653  * Function:    scsi_release_buffers()
654  *
655  * Purpose:     Free resources allocate for a scsi_command.
656  *
657  * Arguments:   cmd     - command that we are bailing.
658  *
659  * Lock status: Assumed that no lock is held upon entry.
660  *
661  * Returns:     Nothing
662  *
663  * Notes:       In the event that an upper level driver rejects a
664  *              command, we must release resources allocated during
665  *              the __init_io() function.  Primarily this would involve
666  *              the scatter-gather table.
667  */
668 static void scsi_release_buffers(struct scsi_cmnd *cmd)
669 {
670         if (cmd->sdb.table.nents)
671                 scsi_free_sgtable(&cmd->sdb, false);
672
673         memset(&cmd->sdb, 0, sizeof(cmd->sdb));
674
675         if (scsi_prot_sg_count(cmd))
676                 scsi_free_sgtable(cmd->prot_sdb, false);
677 }
678
679 static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd)
680 {
681         struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special;
682
683         scsi_free_sgtable(bidi_sdb, false);
684         kmem_cache_free(scsi_sdb_cache, bidi_sdb);
685         cmd->request->next_rq->special = NULL;
686 }
687
688 static bool scsi_end_request(struct request *req, int error,
689                 unsigned int bytes, unsigned int bidi_bytes)
690 {
691         struct scsi_cmnd *cmd = req->special;
692         struct scsi_device *sdev = cmd->device;
693         struct request_queue *q = sdev->request_queue;
694
695         if (blk_update_request(req, error, bytes))
696                 return true;
697
698         /* Bidi request must be completed as a whole */
699         if (unlikely(bidi_bytes) &&
700             blk_update_request(req->next_rq, error, bidi_bytes))
701                 return true;
702
703         if (blk_queue_add_random(q))
704                 add_disk_randomness(req->rq_disk);
705
706         if (req->mq_ctx) {
707                 /*
708                  * In the MQ case the command gets freed by __blk_mq_end_request,
709                  * so we have to do all cleanup that depends on it earlier.
710                  *
711                  * We also can't kick the queues from irq context, so we
712                  * will have to defer it to a workqueue.
713                  */
714                 scsi_mq_uninit_cmd(cmd);
715
716                 __blk_mq_end_request(req, error);
717
718                 if (scsi_target(sdev)->single_lun ||
719                     !list_empty(&sdev->host->starved_list))
720                         kblockd_schedule_work(&sdev->requeue_work);
721                 else
722                         blk_mq_start_stopped_hw_queues(q, true);
723         } else {
724                 unsigned long flags;
725
726                 if (bidi_bytes)
727                         scsi_release_bidi_buffers(cmd);
728
729                 spin_lock_irqsave(q->queue_lock, flags);
730                 blk_finish_request(req, error);
731                 spin_unlock_irqrestore(q->queue_lock, flags);
732
733                 scsi_release_buffers(cmd);
734
735                 scsi_put_command(cmd);
736                 scsi_run_queue(q);
737         }
738
739         put_device(&sdev->sdev_gendev);
740         return false;
741 }
742
743 /**
744  * __scsi_error_from_host_byte - translate SCSI error code into errno
745  * @cmd:        SCSI command (unused)
746  * @result:     scsi error code
747  *
748  * Translate SCSI error code into standard UNIX errno.
749  * Return values:
750  * -ENOLINK     temporary transport failure
751  * -EREMOTEIO   permanent target failure, do not retry
752  * -EBADE       permanent nexus failure, retry on other path
753  * -ENOSPC      No write space available
754  * -ENODATA     Medium error
755  * -EIO         unspecified I/O error
756  */
757 static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result)
758 {
759         int error = 0;
760
761         switch(host_byte(result)) {
762         case DID_TRANSPORT_FAILFAST:
763                 error = -ENOLINK;
764                 break;
765         case DID_TARGET_FAILURE:
766                 set_host_byte(cmd, DID_OK);
767                 error = -EREMOTEIO;
768                 break;
769         case DID_NEXUS_FAILURE:
770                 set_host_byte(cmd, DID_OK);
771                 error = -EBADE;
772                 break;
773         case DID_ALLOC_FAILURE:
774                 set_host_byte(cmd, DID_OK);
775                 error = -ENOSPC;
776                 break;
777         case DID_MEDIUM_ERROR:
778                 set_host_byte(cmd, DID_OK);
779                 error = -ENODATA;
780                 break;
781         default:
782                 error = -EIO;
783                 break;
784         }
785
786         return error;
787 }
788
789 /*
790  * Function:    scsi_io_completion()
791  *
792  * Purpose:     Completion processing for block device I/O requests.
793  *
794  * Arguments:   cmd   - command that is finished.
795  *
796  * Lock status: Assumed that no lock is held upon entry.
797  *
798  * Returns:     Nothing
799  *
800  * Notes:       We will finish off the specified number of sectors.  If we
801  *              are done, the command block will be released and the queue
802  *              function will be goosed.  If we are not done then we have to
803  *              figure out what to do next:
804  *
805  *              a) We can call scsi_requeue_command().  The request
806  *                 will be unprepared and put back on the queue.  Then
807  *                 a new command will be created for it.  This should
808  *                 be used if we made forward progress, or if we want
809  *                 to switch from READ(10) to READ(6) for example.
810  *
811  *              b) We can call __scsi_queue_insert().  The request will
812  *                 be put back on the queue and retried using the same
813  *                 command as before, possibly after a delay.
814  *
815  *              c) We can call scsi_end_request() with -EIO to fail
816  *                 the remainder of the request.
817  */
818 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
819 {
820         int result = cmd->result;
821         struct request_queue *q = cmd->device->request_queue;
822         struct request *req = cmd->request;
823         int error = 0;
824         struct scsi_sense_hdr sshdr;
825         bool sense_valid = false;
826         int sense_deferred = 0, level = 0;
827         enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
828               ACTION_DELAYED_RETRY} action;
829         unsigned long wait_for = (cmd->allowed + 1) * req->timeout;
830
831         if (result) {
832                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
833                 if (sense_valid)
834                         sense_deferred = scsi_sense_is_deferred(&sshdr);
835         }
836
837         if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
838                 if (result) {
839                         if (sense_valid && req->sense) {
840                                 /*
841                                  * SG_IO wants current and deferred errors
842                                  */
843                                 int len = 8 + cmd->sense_buffer[7];
844
845                                 if (len > SCSI_SENSE_BUFFERSIZE)
846                                         len = SCSI_SENSE_BUFFERSIZE;
847                                 memcpy(req->sense, cmd->sense_buffer,  len);
848                                 req->sense_len = len;
849                         }
850                         if (!sense_deferred)
851                                 error = __scsi_error_from_host_byte(cmd, result);
852                 }
853                 /*
854                  * __scsi_error_from_host_byte may have reset the host_byte
855                  */
856                 req->errors = cmd->result;
857
858                 req->resid_len = scsi_get_resid(cmd);
859
860                 if (scsi_bidi_cmnd(cmd)) {
861                         /*
862                          * Bidi commands Must be complete as a whole,
863                          * both sides at once.
864                          */
865                         req->next_rq->resid_len = scsi_in(cmd)->resid;
866                         if (scsi_end_request(req, 0, blk_rq_bytes(req),
867                                         blk_rq_bytes(req->next_rq)))
868                                 BUG();
869                         return;
870                 }
871         } else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) {
872                 /*
873                  * Certain non BLOCK_PC requests are commands that don't
874                  * actually transfer anything (FLUSH), so cannot use
875                  * good_bytes != blk_rq_bytes(req) as the signal for an error.
876                  * This sets the error explicitly for the problem case.
877                  */
878                 error = __scsi_error_from_host_byte(cmd, result);
879         }
880
881         /* no bidi support for !REQ_TYPE_BLOCK_PC yet */
882         BUG_ON(blk_bidi_rq(req));
883
884         /*
885          * Next deal with any sectors which we were able to correctly
886          * handle.
887          */
888         SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,
889                 "%u sectors total, %d bytes done.\n",
890                 blk_rq_sectors(req), good_bytes));
891
892         /*
893          * Recovered errors need reporting, but they're always treated
894          * as success, so fiddle the result code here.  For BLOCK_PC
895          * we already took a copy of the original into rq->errors which
896          * is what gets returned to the user
897          */
898         if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
899                 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
900                  * print since caller wants ATA registers. Only occurs on
901                  * SCSI ATA PASS_THROUGH commands when CK_COND=1
902                  */
903                 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
904                         ;
905                 else if (!(req->cmd_flags & REQ_QUIET))
906                         scsi_print_sense(cmd);
907                 result = 0;
908                 /* BLOCK_PC may have set error */
909                 error = 0;
910         }
911
912         /*
913          * If we finished all bytes in the request we are done now.
914          */
915         if (!scsi_end_request(req, error, good_bytes, 0))
916                 return;
917
918         /*
919          * Kill remainder if no retrys.
920          */
921         if (error && scsi_noretry_cmd(cmd)) {
922                 if (scsi_end_request(req, error, blk_rq_bytes(req), 0))
923                         BUG();
924                 return;
925         }
926
927         /*
928          * If there had been no error, but we have leftover bytes in the
929          * requeues just queue the command up again.
930          */
931         if (result == 0)
932                 goto requeue;
933
934         error = __scsi_error_from_host_byte(cmd, result);
935
936         if (host_byte(result) == DID_RESET) {
937                 /* Third party bus reset or reset for error recovery
938                  * reasons.  Just retry the command and see what
939                  * happens.
940                  */
941                 action = ACTION_RETRY;
942         } else if (sense_valid && !sense_deferred) {
943                 switch (sshdr.sense_key) {
944                 case UNIT_ATTENTION:
945                         if (cmd->device->removable) {
946                                 /* Detected disc change.  Set a bit
947                                  * and quietly refuse further access.
948                                  */
949                                 cmd->device->changed = 1;
950                                 action = ACTION_FAIL;
951                         } else {
952                                 /* Must have been a power glitch, or a
953                                  * bus reset.  Could not have been a
954                                  * media change, so we just retry the
955                                  * command and see what happens.
956                                  */
957                                 action = ACTION_RETRY;
958                         }
959                         break;
960                 case ILLEGAL_REQUEST:
961                         /* If we had an ILLEGAL REQUEST returned, then
962                          * we may have performed an unsupported
963                          * command.  The only thing this should be
964                          * would be a ten byte read where only a six
965                          * byte read was supported.  Also, on a system
966                          * where READ CAPACITY failed, we may have
967                          * read past the end of the disk.
968                          */
969                         if ((cmd->device->use_10_for_rw &&
970                             sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
971                             (cmd->cmnd[0] == READ_10 ||
972                              cmd->cmnd[0] == WRITE_10)) {
973                                 /* This will issue a new 6-byte command. */
974                                 cmd->device->use_10_for_rw = 0;
975                                 action = ACTION_REPREP;
976                         } else if (sshdr.asc == 0x10) /* DIX */ {
977                                 action = ACTION_FAIL;
978                                 error = -EILSEQ;
979                         /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
980                         } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
981                                 action = ACTION_FAIL;
982                                 error = -EREMOTEIO;
983                         } else
984                                 action = ACTION_FAIL;
985                         break;
986                 case ABORTED_COMMAND:
987                         action = ACTION_FAIL;
988                         if (sshdr.asc == 0x10) /* DIF */
989                                 error = -EILSEQ;
990                         break;
991                 case NOT_READY:
992                         /* If the device is in the process of becoming
993                          * ready, or has a temporary blockage, retry.
994                          */
995                         if (sshdr.asc == 0x04) {
996                                 switch (sshdr.ascq) {
997                                 case 0x01: /* becoming ready */
998                                 case 0x04: /* format in progress */
999                                 case 0x05: /* rebuild in progress */
1000                                 case 0x06: /* recalculation in progress */
1001                                 case 0x07: /* operation in progress */
1002                                 case 0x08: /* Long write in progress */
1003                                 case 0x09: /* self test in progress */
1004                                 case 0x14: /* space allocation in progress */
1005                                         action = ACTION_DELAYED_RETRY;
1006                                         break;
1007                                 default:
1008                                         action = ACTION_FAIL;
1009                                         break;
1010                                 }
1011                         } else
1012                                 action = ACTION_FAIL;
1013                         break;
1014                 case VOLUME_OVERFLOW:
1015                         /* See SSC3rXX or current. */
1016                         action = ACTION_FAIL;
1017                         break;
1018                 default:
1019                         action = ACTION_FAIL;
1020                         break;
1021                 }
1022         } else
1023                 action = ACTION_FAIL;
1024
1025         if (action != ACTION_FAIL &&
1026             time_before(cmd->jiffies_at_alloc + wait_for, jiffies))
1027                 action = ACTION_FAIL;
1028
1029         switch (action) {
1030         case ACTION_FAIL:
1031                 /* Give up and fail the remainder of the request */
1032                 if (!(req->cmd_flags & REQ_QUIET)) {
1033                         static DEFINE_RATELIMIT_STATE(_rs,
1034                                         DEFAULT_RATELIMIT_INTERVAL,
1035                                         DEFAULT_RATELIMIT_BURST);
1036
1037                         if (unlikely(scsi_logging_level))
1038                                 level = SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,
1039                                                        SCSI_LOG_MLCOMPLETE_BITS);
1040
1041                         /*
1042                          * if logging is enabled the failure will be printed
1043                          * in scsi_log_completion(), so avoid duplicate messages
1044                          */
1045                         if (!level && __ratelimit(&_rs)) {
1046                                 scsi_print_result(cmd, NULL, FAILED);
1047                                 if (driver_byte(result) & DRIVER_SENSE)
1048                                         scsi_print_sense(cmd);
1049                                 scsi_print_command(cmd);
1050                         }
1051                 }
1052                 if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0))
1053                         return;
1054                 /*FALLTHRU*/
1055         case ACTION_REPREP:
1056         requeue:
1057                 /* Unprep the request and put it back at the head of the queue.
1058                  * A new command will be prepared and issued.
1059                  */
1060                 if (q->mq_ops) {
1061                         cmd->request->cmd_flags &= ~REQ_DONTPREP;
1062                         scsi_mq_uninit_cmd(cmd);
1063                         scsi_mq_requeue_cmd(cmd);
1064                 } else {
1065                         scsi_release_buffers(cmd);
1066                         scsi_requeue_command(q, cmd);
1067                 }
1068                 break;
1069         case ACTION_RETRY:
1070                 /* Retry the same command immediately */
1071                 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
1072                 break;
1073         case ACTION_DELAYED_RETRY:
1074                 /* Retry the same command after a delay */
1075                 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
1076                 break;
1077         }
1078 }
1079
1080 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb)
1081 {
1082         int count;
1083
1084         /*
1085          * If sg table allocation fails, requeue request later.
1086          */
1087         if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
1088                                         req->mq_ctx != NULL)))
1089                 return BLKPREP_DEFER;
1090
1091         /* 
1092          * Next, walk the list, and fill in the addresses and sizes of
1093          * each segment.
1094          */
1095         count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
1096         BUG_ON(count > sdb->table.nents);
1097         sdb->table.nents = count;
1098         sdb->length = blk_rq_bytes(req);
1099         return BLKPREP_OK;
1100 }
1101
1102 /*
1103  * Function:    scsi_init_io()
1104  *
1105  * Purpose:     SCSI I/O initialize function.
1106  *
1107  * Arguments:   cmd   - Command descriptor we wish to initialize
1108  *
1109  * Returns:     0 on success
1110  *              BLKPREP_DEFER if the failure is retryable
1111  *              BLKPREP_KILL if the failure is fatal
1112  */
1113 int scsi_init_io(struct scsi_cmnd *cmd)
1114 {
1115         struct scsi_device *sdev = cmd->device;
1116         struct request *rq = cmd->request;
1117         bool is_mq = (rq->mq_ctx != NULL);
1118         int error;
1119
1120         BUG_ON(!rq->nr_phys_segments);
1121
1122         error = scsi_init_sgtable(rq, &cmd->sdb);
1123         if (error)
1124                 goto err_exit;
1125
1126         if (blk_bidi_rq(rq)) {
1127                 if (!rq->q->mq_ops) {
1128                         struct scsi_data_buffer *bidi_sdb =
1129                                 kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC);
1130                         if (!bidi_sdb) {
1131                                 error = BLKPREP_DEFER;
1132                                 goto err_exit;
1133                         }
1134
1135                         rq->next_rq->special = bidi_sdb;
1136                 }
1137
1138                 error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special);
1139                 if (error)
1140                         goto err_exit;
1141         }
1142
1143         if (blk_integrity_rq(rq)) {
1144                 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
1145                 int ivecs, count;
1146
1147                 BUG_ON(prot_sdb == NULL);
1148                 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
1149
1150                 if (scsi_alloc_sgtable(prot_sdb, ivecs, is_mq)) {
1151                         error = BLKPREP_DEFER;
1152                         goto err_exit;
1153                 }
1154
1155                 count = blk_rq_map_integrity_sg(rq->q, rq->bio,
1156                                                 prot_sdb->table.sgl);
1157                 BUG_ON(unlikely(count > ivecs));
1158                 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
1159
1160                 cmd->prot_sdb = prot_sdb;
1161                 cmd->prot_sdb->table.nents = count;
1162         }
1163
1164         return BLKPREP_OK;
1165 err_exit:
1166         if (is_mq) {
1167                 scsi_mq_free_sgtables(cmd);
1168         } else {
1169                 scsi_release_buffers(cmd);
1170                 cmd->request->special = NULL;
1171                 scsi_put_command(cmd);
1172                 put_device(&sdev->sdev_gendev);
1173         }
1174         return error;
1175 }
1176 EXPORT_SYMBOL(scsi_init_io);
1177
1178 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
1179                 struct request *req)
1180 {
1181         struct scsi_cmnd *cmd;
1182
1183         if (!req->special) {
1184                 /* Bail if we can't get a reference to the device */
1185                 if (!get_device(&sdev->sdev_gendev))
1186                         return NULL;
1187
1188                 cmd = scsi_get_command(sdev, GFP_ATOMIC);
1189                 if (unlikely(!cmd)) {
1190                         put_device(&sdev->sdev_gendev);
1191                         return NULL;
1192                 }
1193                 req->special = cmd;
1194         } else {
1195                 cmd = req->special;
1196         }
1197
1198         /* pull a tag out of the request if we have one */
1199         cmd->tag = req->tag;
1200         cmd->request = req;
1201
1202         cmd->cmnd = req->cmd;
1203         cmd->prot_op = SCSI_PROT_NORMAL;
1204
1205         return cmd;
1206 }
1207
1208 static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
1209 {
1210         struct scsi_cmnd *cmd = req->special;
1211
1212         /*
1213          * BLOCK_PC requests may transfer data, in which case they must
1214          * a bio attached to them.  Or they might contain a SCSI command
1215          * that does not transfer data, in which case they may optionally
1216          * submit a request without an attached bio.
1217          */
1218         if (req->bio) {
1219                 int ret = scsi_init_io(cmd);
1220                 if (unlikely(ret))
1221                         return ret;
1222         } else {
1223                 BUG_ON(blk_rq_bytes(req));
1224
1225                 memset(&cmd->sdb, 0, sizeof(cmd->sdb));
1226         }
1227
1228         cmd->cmd_len = req->cmd_len;
1229         cmd->transfersize = blk_rq_bytes(req);
1230         cmd->allowed = req->retries;
1231         return BLKPREP_OK;
1232 }
1233
1234 /*
1235  * Setup a REQ_TYPE_FS command.  These are simple request from filesystems
1236  * that still need to be translated to SCSI CDBs from the ULD.
1237  */
1238 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
1239 {
1240         struct scsi_cmnd *cmd = req->special;
1241
1242         if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
1243                          && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
1244                 int ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
1245                 if (ret != BLKPREP_OK)
1246                         return ret;
1247         }
1248
1249         memset(cmd->cmnd, 0, BLK_MAX_CDB);
1250         return scsi_cmd_to_driver(cmd)->init_command(cmd);
1251 }
1252
1253 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req)
1254 {
1255         struct scsi_cmnd *cmd = req->special;
1256
1257         if (!blk_rq_bytes(req))
1258                 cmd->sc_data_direction = DMA_NONE;
1259         else if (rq_data_dir(req) == WRITE)
1260                 cmd->sc_data_direction = DMA_TO_DEVICE;
1261         else
1262                 cmd->sc_data_direction = DMA_FROM_DEVICE;
1263
1264         switch (req->cmd_type) {
1265         case REQ_TYPE_FS:
1266                 return scsi_setup_fs_cmnd(sdev, req);
1267         case REQ_TYPE_BLOCK_PC:
1268                 return scsi_setup_blk_pc_cmnd(sdev, req);
1269         default:
1270                 return BLKPREP_KILL;
1271         }
1272 }
1273
1274 static int
1275 scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
1276 {
1277         int ret = BLKPREP_OK;
1278
1279         /*
1280          * If the device is not in running state we will reject some
1281          * or all commands.
1282          */
1283         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1284                 switch (sdev->sdev_state) {
1285                 case SDEV_OFFLINE:
1286                 case SDEV_TRANSPORT_OFFLINE:
1287                         /*
1288                          * If the device is offline we refuse to process any
1289                          * commands.  The device must be brought online
1290                          * before trying any recovery commands.
1291                          */
1292                         sdev_printk(KERN_ERR, sdev,
1293                                     "rejecting I/O to offline device\n");
1294                         ret = BLKPREP_KILL;
1295                         break;
1296                 case SDEV_DEL:
1297                         /*
1298                          * If the device is fully deleted, we refuse to
1299                          * process any commands as well.
1300                          */
1301                         sdev_printk(KERN_ERR, sdev,
1302                                     "rejecting I/O to dead device\n");
1303                         ret = BLKPREP_KILL;
1304                         break;
1305                 case SDEV_QUIESCE:
1306                 case SDEV_BLOCK:
1307                 case SDEV_CREATED_BLOCK:
1308                         /*
1309                          * If the devices is blocked we defer normal commands.
1310                          */
1311                         if (!(req->cmd_flags & REQ_PREEMPT))
1312                                 ret = BLKPREP_DEFER;
1313                         break;
1314                 default:
1315                         /*
1316                          * For any other not fully online state we only allow
1317                          * special commands.  In particular any user initiated
1318                          * command is not allowed.
1319                          */
1320                         if (!(req->cmd_flags & REQ_PREEMPT))
1321                                 ret = BLKPREP_KILL;
1322                         break;
1323                 }
1324         }
1325         return ret;
1326 }
1327
1328 static int
1329 scsi_prep_return(struct request_queue *q, struct request *req, int ret)
1330 {
1331         struct scsi_device *sdev = q->queuedata;
1332
1333         switch (ret) {
1334         case BLKPREP_KILL:
1335                 req->errors = DID_NO_CONNECT << 16;
1336                 /* release the command and kill it */
1337                 if (req->special) {
1338                         struct scsi_cmnd *cmd = req->special;
1339                         scsi_release_buffers(cmd);
1340                         scsi_put_command(cmd);
1341                         put_device(&sdev->sdev_gendev);
1342                         req->special = NULL;
1343                 }
1344                 break;
1345         case BLKPREP_DEFER:
1346                 /*
1347                  * If we defer, the blk_peek_request() returns NULL, but the
1348                  * queue must be restarted, so we schedule a callback to happen
1349                  * shortly.
1350                  */
1351                 if (atomic_read(&sdev->device_busy) == 0)
1352                         blk_delay_queue(q, SCSI_QUEUE_DELAY);
1353                 break;
1354         default:
1355                 req->cmd_flags |= REQ_DONTPREP;
1356         }
1357
1358         return ret;
1359 }
1360
1361 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1362 {
1363         struct scsi_device *sdev = q->queuedata;
1364         struct scsi_cmnd *cmd;
1365         int ret;
1366
1367         ret = scsi_prep_state_check(sdev, req);
1368         if (ret != BLKPREP_OK)
1369                 goto out;
1370
1371         cmd = scsi_get_cmd_from_req(sdev, req);
1372         if (unlikely(!cmd)) {
1373                 ret = BLKPREP_DEFER;
1374                 goto out;
1375         }
1376
1377         ret = scsi_setup_cmnd(sdev, req);
1378 out:
1379         return scsi_prep_return(q, req, ret);
1380 }
1381
1382 static void scsi_unprep_fn(struct request_queue *q, struct request *req)
1383 {
1384         scsi_uninit_cmd(req->special);
1385 }
1386
1387 /*
1388  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1389  * return 0.
1390  *
1391  * Called with the queue_lock held.
1392  */
1393 static inline int scsi_dev_queue_ready(struct request_queue *q,
1394                                   struct scsi_device *sdev)
1395 {
1396         unsigned int busy;
1397
1398         busy = atomic_inc_return(&sdev->device_busy) - 1;
1399         if (atomic_read(&sdev->device_blocked)) {
1400                 if (busy)
1401                         goto out_dec;
1402
1403                 /*
1404                  * unblock after device_blocked iterates to zero
1405                  */
1406                 if (atomic_dec_return(&sdev->device_blocked) > 0) {
1407                         /*
1408                          * For the MQ case we take care of this in the caller.
1409                          */
1410                         if (!q->mq_ops)
1411                                 blk_delay_queue(q, SCSI_QUEUE_DELAY);
1412                         goto out_dec;
1413                 }
1414                 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,
1415                                    "unblocking device at zero depth\n"));
1416         }
1417
1418         if (busy >= sdev->queue_depth)
1419                 goto out_dec;
1420
1421         return 1;
1422 out_dec:
1423         atomic_dec(&sdev->device_busy);
1424         return 0;
1425 }
1426
1427 /*
1428  * scsi_target_queue_ready: checks if there we can send commands to target
1429  * @sdev: scsi device on starget to check.
1430  */
1431 static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
1432                                            struct scsi_device *sdev)
1433 {
1434         struct scsi_target *starget = scsi_target(sdev);
1435         unsigned int busy;
1436
1437         if (starget->single_lun) {
1438                 spin_lock_irq(shost->host_lock);
1439                 if (starget->starget_sdev_user &&
1440                     starget->starget_sdev_user != sdev) {
1441                         spin_unlock_irq(shost->host_lock);
1442                         return 0;
1443                 }
1444                 starget->starget_sdev_user = sdev;
1445                 spin_unlock_irq(shost->host_lock);
1446         }
1447
1448         if (starget->can_queue <= 0)
1449                 return 1;
1450
1451         busy = atomic_inc_return(&starget->target_busy) - 1;
1452         if (atomic_read(&starget->target_blocked) > 0) {
1453                 if (busy)
1454                         goto starved;
1455
1456                 /*
1457                  * unblock after target_blocked iterates to zero
1458                  */
1459                 if (atomic_dec_return(&starget->target_blocked) > 0)
1460                         goto out_dec;
1461
1462                 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
1463                                  "unblocking target at zero depth\n"));
1464         }
1465
1466         if (busy >= starget->can_queue)
1467                 goto starved;
1468
1469         return 1;
1470
1471 starved:
1472         spin_lock_irq(shost->host_lock);
1473         list_move_tail(&sdev->starved_entry, &shost->starved_list);
1474         spin_unlock_irq(shost->host_lock);
1475 out_dec:
1476         if (starget->can_queue > 0)
1477                 atomic_dec(&starget->target_busy);
1478         return 0;
1479 }
1480
1481 /*
1482  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1483  * return 0. We must end up running the queue again whenever 0 is
1484  * returned, else IO can hang.
1485  */
1486 static inline int scsi_host_queue_ready(struct request_queue *q,
1487                                    struct Scsi_Host *shost,
1488                                    struct scsi_device *sdev)
1489 {
1490         unsigned int busy;
1491
1492         if (scsi_host_in_recovery(shost))
1493                 return 0;
1494
1495         busy = atomic_inc_return(&shost->host_busy) - 1;
1496         if (atomic_read(&shost->host_blocked) > 0) {
1497                 if (busy)
1498                         goto starved;
1499
1500                 /*
1501                  * unblock after host_blocked iterates to zero
1502                  */
1503                 if (atomic_dec_return(&shost->host_blocked) > 0)
1504                         goto out_dec;
1505
1506                 SCSI_LOG_MLQUEUE(3,
1507                         shost_printk(KERN_INFO, shost,
1508                                      "unblocking host at zero depth\n"));
1509         }
1510
1511         if (shost->can_queue > 0 && busy >= shost->can_queue)
1512                 goto starved;
1513         if (shost->host_self_blocked)
1514                 goto starved;
1515
1516         /* We're OK to process the command, so we can't be starved */
1517         if (!list_empty(&sdev->starved_entry)) {
1518                 spin_lock_irq(shost->host_lock);
1519                 if (!list_empty(&sdev->starved_entry))
1520                         list_del_init(&sdev->starved_entry);
1521                 spin_unlock_irq(shost->host_lock);
1522         }
1523
1524         return 1;
1525
1526 starved:
1527         spin_lock_irq(shost->host_lock);
1528         if (list_empty(&sdev->starved_entry))
1529                 list_add_tail(&sdev->starved_entry, &shost->starved_list);
1530         spin_unlock_irq(shost->host_lock);
1531 out_dec:
1532         atomic_dec(&shost->host_busy);
1533         return 0;
1534 }
1535
1536 /*
1537  * Busy state exporting function for request stacking drivers.
1538  *
1539  * For efficiency, no lock is taken to check the busy state of
1540  * shost/starget/sdev, since the returned value is not guaranteed and
1541  * may be changed after request stacking drivers call the function,
1542  * regardless of taking lock or not.
1543  *
1544  * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi
1545  * needs to return 'not busy'. Otherwise, request stacking drivers
1546  * may hold requests forever.
1547  */
1548 static int scsi_lld_busy(struct request_queue *q)
1549 {
1550         struct scsi_device *sdev = q->queuedata;
1551         struct Scsi_Host *shost;
1552
1553         if (blk_queue_dying(q))
1554                 return 0;
1555
1556         shost = sdev->host;
1557
1558         /*
1559          * Ignore host/starget busy state.
1560          * Since block layer does not have a concept of fairness across
1561          * multiple queues, congestion of host/starget needs to be handled
1562          * in SCSI layer.
1563          */
1564         if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))
1565                 return 1;
1566
1567         return 0;
1568 }
1569
1570 /*
1571  * Kill a request for a dead device
1572  */
1573 static void scsi_kill_request(struct request *req, struct request_queue *q)
1574 {
1575         struct scsi_cmnd *cmd = req->special;
1576         struct scsi_device *sdev;
1577         struct scsi_target *starget;
1578         struct Scsi_Host *shost;
1579
1580         blk_start_request(req);
1581
1582         scmd_printk(KERN_INFO, cmd, "killing request\n");
1583
1584         sdev = cmd->device;
1585         starget = scsi_target(sdev);
1586         shost = sdev->host;
1587         scsi_init_cmd_errh(cmd);
1588         cmd->result = DID_NO_CONNECT << 16;
1589         atomic_inc(&cmd->device->iorequest_cnt);
1590
1591         /*
1592          * SCSI request completion path will do scsi_device_unbusy(),
1593          * bump busy counts.  To bump the counters, we need to dance
1594          * with the locks as normal issue path does.
1595          */
1596         atomic_inc(&sdev->device_busy);
1597         atomic_inc(&shost->host_busy);
1598         if (starget->can_queue > 0)
1599                 atomic_inc(&starget->target_busy);
1600
1601         blk_complete_request(req);
1602 }
1603
1604 static void scsi_softirq_done(struct request *rq)
1605 {
1606         struct scsi_cmnd *cmd = rq->special;
1607         unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
1608         int disposition;
1609
1610         INIT_LIST_HEAD(&cmd->eh_entry);
1611
1612         atomic_inc(&cmd->device->iodone_cnt);
1613         if (cmd->result)
1614                 atomic_inc(&cmd->device->ioerr_cnt);
1615
1616         disposition = scsi_decide_disposition(cmd);
1617         if (disposition != SUCCESS &&
1618             time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1619                 sdev_printk(KERN_ERR, cmd->device,
1620                             "timing out command, waited %lus\n",
1621                             wait_for/HZ);
1622                 disposition = SUCCESS;
1623         }
1624
1625         scsi_log_completion(cmd, disposition);
1626
1627         switch (disposition) {
1628                 case SUCCESS:
1629                         scsi_finish_command(cmd);
1630                         break;
1631                 case NEEDS_RETRY:
1632                         scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
1633                         break;
1634                 case ADD_TO_MLQUEUE:
1635                         scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1636                         break;
1637                 default:
1638                         if (!scsi_eh_scmd_add(cmd, 0))
1639                                 scsi_finish_command(cmd);
1640         }
1641 }
1642
1643 /**
1644  * scsi_done - Invoke completion on finished SCSI command.
1645  * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives
1646  * ownership back to SCSI Core -- i.e. the LLDD has finished with it.
1647  *
1648  * Description: This function is the mid-level's (SCSI Core) interrupt routine,
1649  * which regains ownership of the SCSI command (de facto) from a LLDD, and
1650  * calls blk_complete_request() for further processing.
1651  *
1652  * This function is interrupt context safe.
1653  */
1654 static void scsi_done(struct scsi_cmnd *cmd)
1655 {
1656         trace_scsi_dispatch_cmd_done(cmd);
1657         blk_complete_request(cmd->request);
1658 }
1659
1660 /*
1661  * Function:    scsi_request_fn()
1662  *
1663  * Purpose:     Main strategy routine for SCSI.
1664  *
1665  * Arguments:   q       - Pointer to actual queue.
1666  *
1667  * Returns:     Nothing
1668  *
1669  * Lock status: IO request lock assumed to be held when called.
1670  */
1671 static void scsi_request_fn(struct request_queue *q)
1672         __releases(q->queue_lock)
1673         __acquires(q->queue_lock)
1674 {
1675         struct scsi_device *sdev = q->queuedata;
1676         struct Scsi_Host *shost;
1677         struct scsi_cmnd *cmd;
1678         struct request *req;
1679
1680         /*
1681          * To start with, we keep looping until the queue is empty, or until
1682          * the host is no longer able to accept any more requests.
1683          */
1684         shost = sdev->host;
1685         for (;;) {
1686                 int rtn;
1687                 /*
1688                  * get next queueable request.  We do this early to make sure
1689                  * that the request is fully prepared even if we cannot
1690                  * accept it.
1691                  */
1692                 req = blk_peek_request(q);
1693                 if (!req)
1694                         break;
1695
1696                 if (unlikely(!scsi_device_online(sdev))) {
1697                         sdev_printk(KERN_ERR, sdev,
1698                                     "rejecting I/O to offline device\n");
1699                         scsi_kill_request(req, q);
1700                         continue;
1701                 }
1702
1703                 if (!scsi_dev_queue_ready(q, sdev))
1704                         break;
1705
1706                 /*
1707                  * Remove the request from the request list.
1708                  */
1709                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1710                         blk_start_request(req);
1711
1712                 spin_unlock_irq(q->queue_lock);
1713                 cmd = req->special;
1714                 if (unlikely(cmd == NULL)) {
1715                         printk(KERN_CRIT "impossible request in %s.\n"
1716                                          "please mail a stack trace to "
1717                                          "linux-scsi@vger.kernel.org\n",
1718                                          __func__);
1719                         blk_dump_rq_flags(req, "foo");
1720                         BUG();
1721                 }
1722
1723                 /*
1724                  * We hit this when the driver is using a host wide
1725                  * tag map. For device level tag maps the queue_depth check
1726                  * in the device ready fn would prevent us from trying
1727                  * to allocate a tag. Since the map is a shared host resource
1728                  * we add the dev to the starved list so it eventually gets
1729                  * a run when a tag is freed.
1730                  */
1731                 if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) {
1732                         spin_lock_irq(shost->host_lock);
1733                         if (list_empty(&sdev->starved_entry))
1734                                 list_add_tail(&sdev->starved_entry,
1735                                               &shost->starved_list);
1736                         spin_unlock_irq(shost->host_lock);
1737                         goto not_ready;
1738                 }
1739
1740                 if (!scsi_target_queue_ready(shost, sdev))
1741                         goto not_ready;
1742
1743                 if (!scsi_host_queue_ready(q, shost, sdev))
1744                         goto host_not_ready;
1745         
1746                 if (sdev->simple_tags)
1747                         cmd->flags |= SCMD_TAGGED;
1748                 else
1749                         cmd->flags &= ~SCMD_TAGGED;
1750
1751                 /*
1752                  * Finally, initialize any error handling parameters, and set up
1753                  * the timers for timeouts.
1754                  */
1755                 scsi_init_cmd_errh(cmd);
1756
1757                 /*
1758                  * Dispatch the command to the low-level driver.
1759                  */
1760                 cmd->scsi_done = scsi_done;
1761                 rtn = scsi_dispatch_cmd(cmd);
1762                 if (rtn) {
1763                         scsi_queue_insert(cmd, rtn);
1764                         spin_lock_irq(q->queue_lock);
1765                         goto out_delay;
1766                 }
1767                 spin_lock_irq(q->queue_lock);
1768         }
1769
1770         return;
1771
1772  host_not_ready:
1773         if (scsi_target(sdev)->can_queue > 0)
1774                 atomic_dec(&scsi_target(sdev)->target_busy);
1775  not_ready:
1776         /*
1777          * lock q, handle tag, requeue req, and decrement device_busy. We
1778          * must return with queue_lock held.
1779          *
1780          * Decrementing device_busy without checking it is OK, as all such
1781          * cases (host limits or settings) should run the queue at some
1782          * later time.
1783          */
1784         spin_lock_irq(q->queue_lock);
1785         blk_requeue_request(q, req);
1786         atomic_dec(&sdev->device_busy);
1787 out_delay:
1788         if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev))
1789                 blk_delay_queue(q, SCSI_QUEUE_DELAY);
1790 }
1791
1792 static inline int prep_to_mq(int ret)
1793 {
1794         switch (ret) {
1795         case BLKPREP_OK:
1796                 return 0;
1797         case BLKPREP_DEFER:
1798                 return BLK_MQ_RQ_QUEUE_BUSY;
1799         default:
1800                 return BLK_MQ_RQ_QUEUE_ERROR;
1801         }
1802 }
1803
1804 static int scsi_mq_prep_fn(struct request *req)
1805 {
1806         struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1807         struct scsi_device *sdev = req->q->queuedata;
1808         struct Scsi_Host *shost = sdev->host;
1809         unsigned char *sense_buf = cmd->sense_buffer;
1810         struct scatterlist *sg;
1811
1812         memset(cmd, 0, sizeof(struct scsi_cmnd));
1813
1814         req->special = cmd;
1815
1816         cmd->request = req;
1817         cmd->device = sdev;
1818         cmd->sense_buffer = sense_buf;
1819
1820         cmd->tag = req->tag;
1821
1822         cmd->cmnd = req->cmd;
1823         cmd->prot_op = SCSI_PROT_NORMAL;
1824
1825         INIT_LIST_HEAD(&cmd->list);
1826         INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
1827         cmd->jiffies_at_alloc = jiffies;
1828
1829         if (shost->use_cmd_list) {
1830                 spin_lock_irq(&sdev->list_lock);
1831                 list_add_tail(&cmd->list, &sdev->cmd_list);
1832                 spin_unlock_irq(&sdev->list_lock);
1833         }
1834
1835         sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
1836         cmd->sdb.table.sgl = sg;
1837
1838         if (scsi_host_get_prot(shost)) {
1839                 cmd->prot_sdb = (void *)sg +
1840                         shost->sg_tablesize * sizeof(struct scatterlist);
1841                 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer));
1842
1843                 cmd->prot_sdb->table.sgl =
1844                         (struct scatterlist *)(cmd->prot_sdb + 1);
1845         }
1846
1847         if (blk_bidi_rq(req)) {
1848                 struct request *next_rq = req->next_rq;
1849                 struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq);
1850
1851                 memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer));
1852                 bidi_sdb->table.sgl =
1853                         (struct scatterlist *)(bidi_sdb + 1);
1854
1855                 next_rq->special = bidi_sdb;
1856         }
1857
1858         blk_mq_start_request(req);
1859
1860         return scsi_setup_cmnd(sdev, req);
1861 }
1862
1863 static void scsi_mq_done(struct scsi_cmnd *cmd)
1864 {
1865         trace_scsi_dispatch_cmd_done(cmd);
1866         blk_mq_complete_request(cmd->request);
1867 }
1868
1869 static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req,
1870                 bool last)
1871 {
1872         struct request_queue *q = req->q;
1873         struct scsi_device *sdev = q->queuedata;
1874         struct Scsi_Host *shost = sdev->host;
1875         struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1876         int ret;
1877         int reason;
1878
1879         ret = prep_to_mq(scsi_prep_state_check(sdev, req));
1880         if (ret)
1881                 goto out;
1882
1883         ret = BLK_MQ_RQ_QUEUE_BUSY;
1884         if (!get_device(&sdev->sdev_gendev))
1885                 goto out;
1886
1887         if (!scsi_dev_queue_ready(q, sdev))
1888                 goto out_put_device;
1889         if (!scsi_target_queue_ready(shost, sdev))
1890                 goto out_dec_device_busy;
1891         if (!scsi_host_queue_ready(q, shost, sdev))
1892                 goto out_dec_target_busy;
1893
1894
1895         if (!(req->cmd_flags & REQ_DONTPREP)) {
1896                 ret = prep_to_mq(scsi_mq_prep_fn(req));
1897                 if (ret)
1898                         goto out_dec_host_busy;
1899                 req->cmd_flags |= REQ_DONTPREP;
1900         } else {
1901                 blk_mq_start_request(req);
1902         }
1903
1904         if (sdev->simple_tags)
1905                 cmd->flags |= SCMD_TAGGED;
1906         else
1907                 cmd->flags &= ~SCMD_TAGGED;
1908
1909         scsi_init_cmd_errh(cmd);
1910         cmd->scsi_done = scsi_mq_done;
1911
1912         reason = scsi_dispatch_cmd(cmd);
1913         if (reason) {
1914                 scsi_set_blocked(cmd, reason);
1915                 ret = BLK_MQ_RQ_QUEUE_BUSY;
1916                 goto out_dec_host_busy;
1917         }
1918
1919         return BLK_MQ_RQ_QUEUE_OK;
1920
1921 out_dec_host_busy:
1922         atomic_dec(&shost->host_busy);
1923 out_dec_target_busy:
1924         if (scsi_target(sdev)->can_queue > 0)
1925                 atomic_dec(&scsi_target(sdev)->target_busy);
1926 out_dec_device_busy:
1927         atomic_dec(&sdev->device_busy);
1928 out_put_device:
1929         put_device(&sdev->sdev_gendev);
1930 out:
1931         switch (ret) {
1932         case BLK_MQ_RQ_QUEUE_BUSY:
1933                 blk_mq_stop_hw_queue(hctx);
1934                 if (atomic_read(&sdev->device_busy) == 0 &&
1935                     !scsi_device_blocked(sdev))
1936                         blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY);
1937                 break;
1938         case BLK_MQ_RQ_QUEUE_ERROR:
1939                 /*
1940                  * Make sure to release all allocated ressources when
1941                  * we hit an error, as we will never see this command
1942                  * again.
1943                  */
1944                 if (req->cmd_flags & REQ_DONTPREP)
1945                         scsi_mq_uninit_cmd(cmd);
1946                 break;
1947         default:
1948                 break;
1949         }
1950         return ret;
1951 }
1952
1953 static enum blk_eh_timer_return scsi_timeout(struct request *req,
1954                 bool reserved)
1955 {
1956         if (reserved)
1957                 return BLK_EH_RESET_TIMER;
1958         return scsi_times_out(req);
1959 }
1960
1961 static int scsi_init_request(void *data, struct request *rq,
1962                 unsigned int hctx_idx, unsigned int request_idx,
1963                 unsigned int numa_node)
1964 {
1965         struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
1966
1967         cmd->sense_buffer = kzalloc_node(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL,
1968                         numa_node);
1969         if (!cmd->sense_buffer)
1970                 return -ENOMEM;
1971         return 0;
1972 }
1973
1974 static void scsi_exit_request(void *data, struct request *rq,
1975                 unsigned int hctx_idx, unsigned int request_idx)
1976 {
1977         struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
1978
1979         kfree(cmd->sense_buffer);
1980 }
1981
1982 static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1983 {
1984         struct device *host_dev;
1985         u64 bounce_limit = 0xffffffff;
1986
1987         if (shost->unchecked_isa_dma)
1988                 return BLK_BOUNCE_ISA;
1989         /*
1990          * Platforms with virtual-DMA translation
1991          * hardware have no practical limit.
1992          */
1993         if (!PCI_DMA_BUS_IS_PHYS)
1994                 return BLK_BOUNCE_ANY;
1995
1996         host_dev = scsi_get_device(shost);
1997         if (host_dev && host_dev->dma_mask)
1998                 bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT;
1999
2000         return bounce_limit;
2001 }
2002
2003 static void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
2004 {
2005         struct device *dev = shost->dma_dev;
2006
2007         /*
2008          * this limit is imposed by hardware restrictions
2009          */
2010         blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
2011                                         SCSI_MAX_SG_CHAIN_SEGMENTS));
2012
2013         if (scsi_host_prot_dma(shost)) {
2014                 shost->sg_prot_tablesize =
2015                         min_not_zero(shost->sg_prot_tablesize,
2016                                      (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
2017                 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
2018                 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
2019         }
2020
2021         blk_queue_max_hw_sectors(q, shost->max_sectors);
2022         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
2023         blk_queue_segment_boundary(q, shost->dma_boundary);
2024         dma_set_seg_boundary(dev, shost->dma_boundary);
2025
2026         blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
2027
2028         if (!shost->use_clustering)
2029                 q->limits.cluster = 0;
2030
2031         /*
2032          * set a reasonable default alignment on word boundaries: the
2033          * host and device may alter it using
2034          * blk_queue_update_dma_alignment() later.
2035          */
2036         blk_queue_dma_alignment(q, 0x03);
2037 }
2038
2039 struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
2040                                          request_fn_proc *request_fn)
2041 {
2042         struct request_queue *q;
2043
2044         q = blk_init_queue(request_fn, NULL);
2045         if (!q)
2046                 return NULL;
2047         __scsi_init_queue(shost, q);
2048         return q;
2049 }
2050 EXPORT_SYMBOL(__scsi_alloc_queue);
2051
2052 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
2053 {
2054         struct request_queue *q;
2055
2056         q = __scsi_alloc_queue(sdev->host, scsi_request_fn);
2057         if (!q)
2058                 return NULL;
2059
2060         blk_queue_prep_rq(q, scsi_prep_fn);
2061         blk_queue_unprep_rq(q, scsi_unprep_fn);
2062         blk_queue_softirq_done(q, scsi_softirq_done);
2063         blk_queue_rq_timed_out(q, scsi_times_out);
2064         blk_queue_lld_busy(q, scsi_lld_busy);
2065         return q;
2066 }
2067
2068 static struct blk_mq_ops scsi_mq_ops = {
2069         .map_queue      = blk_mq_map_queue,
2070         .queue_rq       = scsi_queue_rq,
2071         .complete       = scsi_softirq_done,
2072         .timeout        = scsi_timeout,
2073         .init_request   = scsi_init_request,
2074         .exit_request   = scsi_exit_request,
2075 };
2076
2077 struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev)
2078 {
2079         sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set);
2080         if (IS_ERR(sdev->request_queue))
2081                 return NULL;
2082
2083         sdev->request_queue->queuedata = sdev;
2084         __scsi_init_queue(sdev->host, sdev->request_queue);
2085         return sdev->request_queue;
2086 }
2087
2088 int scsi_mq_setup_tags(struct Scsi_Host *shost)
2089 {
2090         unsigned int cmd_size, sgl_size, tbl_size;
2091
2092         tbl_size = shost->sg_tablesize;
2093         if (tbl_size > SCSI_MAX_SG_SEGMENTS)
2094                 tbl_size = SCSI_MAX_SG_SEGMENTS;
2095         sgl_size = tbl_size * sizeof(struct scatterlist);
2096         cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size;
2097         if (scsi_host_get_prot(shost))
2098                 cmd_size += sizeof(struct scsi_data_buffer) + sgl_size;
2099
2100         memset(&shost->tag_set, 0, sizeof(shost->tag_set));
2101         shost->tag_set.ops = &scsi_mq_ops;
2102         shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1;
2103         shost->tag_set.queue_depth = shost->can_queue;
2104         shost->tag_set.cmd_size = cmd_size;
2105         shost->tag_set.numa_node = NUMA_NO_NODE;
2106         shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
2107         shost->tag_set.driver_data = shost;
2108
2109         return blk_mq_alloc_tag_set(&shost->tag_set);
2110 }
2111
2112 void scsi_mq_destroy_tags(struct Scsi_Host *shost)
2113 {
2114         blk_mq_free_tag_set(&shost->tag_set);
2115 }
2116
2117 /*
2118  * Function:    scsi_block_requests()
2119  *
2120  * Purpose:     Utility function used by low-level drivers to prevent further
2121  *              commands from being queued to the device.
2122  *
2123  * Arguments:   shost       - Host in question
2124  *
2125  * Returns:     Nothing
2126  *
2127  * Lock status: No locks are assumed held.
2128  *
2129  * Notes:       There is no timer nor any other means by which the requests
2130  *              get unblocked other than the low-level driver calling
2131  *              scsi_unblock_requests().
2132  */
2133 void scsi_block_requests(struct Scsi_Host *shost)
2134 {
2135         shost->host_self_blocked = 1;
2136 }
2137 EXPORT_SYMBOL(scsi_block_requests);
2138
2139 /*
2140  * Function:    scsi_unblock_requests()
2141  *
2142  * Purpose:     Utility function used by low-level drivers to allow further
2143  *              commands from being queued to the device.
2144  *
2145  * Arguments:   shost       - Host in question
2146  *
2147  * Returns:     Nothing
2148  *
2149  * Lock status: No locks are assumed held.
2150  *
2151  * Notes:       There is no timer nor any other means by which the requests
2152  *              get unblocked other than the low-level driver calling
2153  *              scsi_unblock_requests().
2154  *
2155  *              This is done as an API function so that changes to the
2156  *              internals of the scsi mid-layer won't require wholesale
2157  *              changes to drivers that use this feature.
2158  */
2159 void scsi_unblock_requests(struct Scsi_Host *shost)
2160 {
2161         shost->host_self_blocked = 0;
2162         scsi_run_host_queues(shost);
2163 }
2164 EXPORT_SYMBOL(scsi_unblock_requests);
2165
2166 int __init scsi_init_queue(void)
2167 {
2168         int i;
2169
2170         scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
2171                                            sizeof(struct scsi_data_buffer),
2172                                            0, 0, NULL);
2173         if (!scsi_sdb_cache) {
2174                 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
2175                 return -ENOMEM;
2176         }
2177
2178         for (i = 0; i < SG_MEMPOOL_NR; i++) {
2179                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
2180                 int size = sgp->size * sizeof(struct scatterlist);
2181
2182                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
2183                                 SLAB_HWCACHE_ALIGN, NULL);
2184                 if (!sgp->slab) {
2185                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
2186                                         sgp->name);
2187                         goto cleanup_sdb;
2188                 }
2189
2190                 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
2191                                                      sgp->slab);
2192                 if (!sgp->pool) {
2193                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
2194                                         sgp->name);
2195                         goto cleanup_sdb;
2196                 }
2197         }
2198
2199         return 0;
2200
2201 cleanup_sdb:
2202         for (i = 0; i < SG_MEMPOOL_NR; i++) {
2203                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
2204                 if (sgp->pool)
2205                         mempool_destroy(sgp->pool);
2206                 if (sgp->slab)
2207                         kmem_cache_destroy(sgp->slab);
2208         }
2209         kmem_cache_destroy(scsi_sdb_cache);
2210
2211         return -ENOMEM;
2212 }
2213
2214 void scsi_exit_queue(void)
2215 {
2216         int i;
2217
2218         kmem_cache_destroy(scsi_sdb_cache);
2219
2220         for (i = 0; i < SG_MEMPOOL_NR; i++) {
2221                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
2222                 mempool_destroy(sgp->pool);
2223                 kmem_cache_destroy(sgp->slab);
2224         }
2225 }
2226
2227 /**
2228  *      scsi_mode_select - issue a mode select
2229  *      @sdev:  SCSI device to be queried
2230  *      @pf:    Page format bit (1 == standard, 0 == vendor specific)
2231  *      @sp:    Save page bit (0 == don't save, 1 == save)
2232  *      @modepage: mode page being requested
2233  *      @buffer: request buffer (may not be smaller than eight bytes)
2234  *      @len:   length of request buffer.
2235  *      @timeout: command timeout
2236  *      @retries: number of retries before failing
2237  *      @data: returns a structure abstracting the mode header data
2238  *      @sshdr: place to put sense data (or NULL if no sense to be collected).
2239  *              must be SCSI_SENSE_BUFFERSIZE big.
2240  *
2241  *      Returns zero if successful; negative error number or scsi
2242  *      status on error
2243  *
2244  */
2245 int
2246 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
2247                  unsigned char *buffer, int len, int timeout, int retries,
2248                  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
2249 {
2250         unsigned char cmd[10];
2251         unsigned char *real_buffer;
2252         int ret;
2253
2254         memset(cmd, 0, sizeof(cmd));
2255         cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
2256
2257         if (sdev->use_10_for_ms) {
2258                 if (len > 65535)
2259                         return -EINVAL;
2260                 real_buffer = kmalloc(8 + len, GFP_KERNEL);
2261                 if (!real_buffer)
2262                         return -ENOMEM;
2263                 memcpy(real_buffer + 8, buffer, len);
2264                 len += 8;
2265                 real_buffer[0] = 0;
2266                 real_buffer[1] = 0;
2267                 real_buffer[2] = data->medium_type;
2268                 real_buffer[3] = data->device_specific;
2269                 real_buffer[4] = data->longlba ? 0x01 : 0;
2270                 real_buffer[5] = 0;
2271                 real_buffer[6] = data->block_descriptor_length >> 8;
2272                 real_buffer[7] = data->block_descriptor_length;
2273
2274                 cmd[0] = MODE_SELECT_10;
2275                 cmd[7] = len >> 8;
2276                 cmd[8] = len;
2277         } else {
2278                 if (len > 255 || data->block_descriptor_length > 255 ||
2279                     data->longlba)
2280                         return -EINVAL;
2281
2282                 real_buffer = kmalloc(4 + len, GFP_KERNEL);
2283                 if (!real_buffer)
2284                         return -ENOMEM;
2285                 memcpy(real_buffer + 4, buffer, len);
2286                 len += 4;
2287                 real_buffer[0] = 0;
2288                 real_buffer[1] = data->medium_type;
2289                 real_buffer[2] = data->device_specific;
2290                 real_buffer[3] = data->block_descriptor_length;
2291                 
2292
2293                 cmd[0] = MODE_SELECT;
2294                 cmd[4] = len;
2295         }
2296
2297         ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
2298                                sshdr, timeout, retries, NULL);
2299         kfree(real_buffer);
2300         return ret;
2301 }
2302 EXPORT_SYMBOL_GPL(scsi_mode_select);
2303
2304 /**
2305  *      scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
2306  *      @sdev:  SCSI device to be queried
2307  *      @dbd:   set if mode sense will allow block descriptors to be returned
2308  *      @modepage: mode page being requested
2309  *      @buffer: request buffer (may not be smaller than eight bytes)
2310  *      @len:   length of request buffer.
2311  *      @timeout: command timeout
2312  *      @retries: number of retries before failing
2313  *      @data: returns a structure abstracting the mode header data
2314  *      @sshdr: place to put sense data (or NULL if no sense to be collected).
2315  *              must be SCSI_SENSE_BUFFERSIZE big.
2316  *
2317  *      Returns zero if unsuccessful, or the header offset (either 4
2318  *      or 8 depending on whether a six or ten byte command was
2319  *      issued) if successful.
2320  */
2321 int
2322 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
2323                   unsigned char *buffer, int len, int timeout, int retries,
2324                   struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
2325 {
2326         unsigned char cmd[12];
2327         int use_10_for_ms;
2328         int header_length;
2329         int result;
2330         struct scsi_sense_hdr my_sshdr;
2331
2332         memset(data, 0, sizeof(*data));
2333         memset(&cmd[0], 0, 12);
2334         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
2335         cmd[2] = modepage;
2336
2337         /* caller might not be interested in sense, but we need it */
2338         if (!sshdr)
2339                 sshdr = &my_sshdr;
2340
2341  retry:
2342         use_10_for_ms = sdev->use_10_for_ms;
2343
2344         if (use_10_for_ms) {
2345                 if (len < 8)
2346                         len = 8;
2347
2348                 cmd[0] = MODE_SENSE_10;
2349                 cmd[8] = len;
2350                 header_length = 8;
2351         } else {
2352                 if (len < 4)
2353                         len = 4;
2354
2355                 cmd[0] = MODE_SENSE;
2356                 cmd[4] = len;
2357                 header_length = 4;
2358         }
2359
2360         memset(buffer, 0, len);
2361
2362         result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
2363                                   sshdr, timeout, retries, NULL);
2364
2365         /* This code looks awful: what it's doing is making sure an
2366          * ILLEGAL REQUEST sense return identifies the actual command
2367          * byte as the problem.  MODE_SENSE commands can return
2368          * ILLEGAL REQUEST if the code page isn't supported */
2369
2370         if (use_10_for_ms && !scsi_status_is_good(result) &&
2371             (driver_byte(result) & DRIVER_SENSE)) {
2372                 if (scsi_sense_valid(sshdr)) {
2373                         if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
2374                             (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
2375                                 /* 
2376                                  * Invalid command operation code
2377                                  */
2378                                 sdev->use_10_for_ms = 0;
2379                                 goto retry;
2380                         }
2381                 }
2382         }
2383
2384         if(scsi_status_is_good(result)) {
2385                 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
2386                              (modepage == 6 || modepage == 8))) {
2387                         /* Initio breakage? */
2388                         header_length = 0;
2389                         data->length = 13;
2390                         data->medium_type = 0;
2391                         data->device_specific = 0;
2392                         data->longlba = 0;
2393                         data->block_descriptor_length = 0;
2394                 } else if(use_10_for_ms) {
2395                         data->length = buffer[0]*256 + buffer[1] + 2;
2396                         data->medium_type = buffer[2];
2397                         data->device_specific = buffer[3];
2398                         data->longlba = buffer[4] & 0x01;
2399                         data->block_descriptor_length = buffer[6]*256
2400                                 + buffer[7];
2401                 } else {
2402                         data->length = buffer[0] + 1;
2403                         data->medium_type = buffer[1];
2404                         data->device_specific = buffer[2];
2405                         data->block_descriptor_length = buffer[3];
2406                 }
2407                 data->header_length = header_length;
2408         }
2409
2410         return result;
2411 }
2412 EXPORT_SYMBOL(scsi_mode_sense);
2413
2414 /**
2415  *      scsi_test_unit_ready - test if unit is ready
2416  *      @sdev:  scsi device to change the state of.
2417  *      @timeout: command timeout
2418  *      @retries: number of retries before failing
2419  *      @sshdr_external: Optional pointer to struct scsi_sense_hdr for
2420  *              returning sense. Make sure that this is cleared before passing
2421  *              in.
2422  *
2423  *      Returns zero if unsuccessful or an error if TUR failed.  For
2424  *      removable media, UNIT_ATTENTION sets ->changed flag.
2425  **/
2426 int
2427 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
2428                      struct scsi_sense_hdr *sshdr_external)
2429 {
2430         char cmd[] = {
2431                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
2432         };
2433         struct scsi_sense_hdr *sshdr;
2434         int result;
2435
2436         if (!sshdr_external)
2437                 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
2438         else
2439                 sshdr = sshdr_external;
2440
2441         /* try to eat the UNIT_ATTENTION if there are enough retries */
2442         do {
2443                 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
2444                                           timeout, retries, NULL);
2445                 if (sdev->removable && scsi_sense_valid(sshdr) &&
2446                     sshdr->sense_key == UNIT_ATTENTION)
2447                         sdev->changed = 1;
2448         } while (scsi_sense_valid(sshdr) &&
2449                  sshdr->sense_key == UNIT_ATTENTION && --retries);
2450
2451         if (!sshdr_external)
2452                 kfree(sshdr);
2453         return result;
2454 }
2455 EXPORT_SYMBOL(scsi_test_unit_ready);
2456
2457 /**
2458  *      scsi_device_set_state - Take the given device through the device state model.
2459  *      @sdev:  scsi device to change the state of.
2460  *      @state: state to change to.
2461  *
2462  *      Returns zero if unsuccessful or an error if the requested 
2463  *      transition is illegal.
2464  */
2465 int
2466 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
2467 {
2468         enum scsi_device_state oldstate = sdev->sdev_state;
2469
2470         if (state == oldstate)
2471                 return 0;
2472
2473         switch (state) {
2474         case SDEV_CREATED:
2475                 switch (oldstate) {
2476                 case SDEV_CREATED_BLOCK:
2477                         break;
2478                 default:
2479                         goto illegal;
2480                 }
2481                 break;
2482                         
2483         case SDEV_RUNNING:
2484                 switch (oldstate) {
2485                 case SDEV_CREATED:
2486                 case SDEV_OFFLINE:
2487                 case SDEV_TRANSPORT_OFFLINE:
2488                 case SDEV_QUIESCE:
2489                 case SDEV_BLOCK:
2490                         break;
2491                 default:
2492                         goto illegal;
2493                 }
2494                 break;
2495
2496         case SDEV_QUIESCE:
2497                 switch (oldstate) {
2498                 case SDEV_RUNNING:
2499                 case SDEV_OFFLINE:
2500                 case SDEV_TRANSPORT_OFFLINE:
2501                         break;
2502                 default:
2503                         goto illegal;
2504                 }
2505                 break;
2506
2507         case SDEV_OFFLINE:
2508         case SDEV_TRANSPORT_OFFLINE:
2509                 switch (oldstate) {
2510                 case SDEV_CREATED:
2511                 case SDEV_RUNNING:
2512                 case SDEV_QUIESCE:
2513                 case SDEV_BLOCK:
2514                         break;
2515                 default:
2516                         goto illegal;
2517                 }
2518                 break;
2519
2520         case SDEV_BLOCK:
2521                 switch (oldstate) {
2522                 case SDEV_RUNNING:
2523                 case SDEV_CREATED_BLOCK:
2524                         break;
2525                 default:
2526                         goto illegal;
2527                 }
2528                 break;
2529
2530         case SDEV_CREATED_BLOCK:
2531                 switch (oldstate) {
2532                 case SDEV_CREATED:
2533                         break;
2534                 default:
2535                         goto illegal;
2536                 }
2537                 break;
2538
2539         case SDEV_CANCEL:
2540                 switch (oldstate) {
2541                 case SDEV_CREATED:
2542                 case SDEV_RUNNING:
2543                 case SDEV_QUIESCE:
2544                 case SDEV_OFFLINE:
2545                 case SDEV_TRANSPORT_OFFLINE:
2546                 case SDEV_BLOCK:
2547                         break;
2548                 default:
2549                         goto illegal;
2550                 }
2551                 break;
2552
2553         case SDEV_DEL:
2554                 switch (oldstate) {
2555                 case SDEV_CREATED:
2556                 case SDEV_RUNNING:
2557                 case SDEV_OFFLINE:
2558                 case SDEV_TRANSPORT_OFFLINE:
2559                 case SDEV_CANCEL:
2560                 case SDEV_CREATED_BLOCK:
2561                         break;
2562                 default:
2563                         goto illegal;
2564                 }
2565                 break;
2566
2567         }
2568         sdev->sdev_state = state;
2569         return 0;
2570
2571  illegal:
2572         SCSI_LOG_ERROR_RECOVERY(1,
2573                                 sdev_printk(KERN_ERR, sdev,
2574                                             "Illegal state transition %s->%s",
2575                                             scsi_device_state_name(oldstate),
2576                                             scsi_device_state_name(state))
2577                                 );
2578         return -EINVAL;
2579 }
2580 EXPORT_SYMBOL(scsi_device_set_state);
2581
2582 /**
2583  *      sdev_evt_emit - emit a single SCSI device uevent
2584  *      @sdev: associated SCSI device
2585  *      @evt: event to emit
2586  *
2587  *      Send a single uevent (scsi_event) to the associated scsi_device.
2588  */
2589 static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)
2590 {
2591         int idx = 0;
2592         char *envp[3];
2593
2594         switch (evt->evt_type) {
2595         case SDEV_EVT_MEDIA_CHANGE:
2596                 envp[idx++] = "SDEV_MEDIA_CHANGE=1";
2597                 break;
2598         case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
2599                 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED";
2600                 break;
2601         case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
2602                 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED";
2603                 break;
2604         case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
2605                envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED";
2606                 break;
2607         case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
2608                 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED";
2609                 break;
2610         case SDEV_EVT_LUN_CHANGE_REPORTED:
2611                 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED";
2612                 break;
2613         default:
2614                 /* do nothing */
2615                 break;
2616         }
2617
2618         envp[idx++] = NULL;
2619
2620         kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);
2621 }
2622
2623 /**
2624  *      sdev_evt_thread - send a uevent for each scsi event
2625  *      @work: work struct for scsi_device
2626  *
2627  *      Dispatch queued events to their associated scsi_device kobjects
2628  *      as uevents.
2629  */
2630 void scsi_evt_thread(struct work_struct *work)
2631 {
2632         struct scsi_device *sdev;
2633         enum scsi_device_event evt_type;
2634         LIST_HEAD(event_list);
2635
2636         sdev = container_of(work, struct scsi_device, event_work);
2637
2638         for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++)
2639                 if (test_and_clear_bit(evt_type, sdev->pending_events))
2640                         sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL);
2641
2642         while (1) {
2643                 struct scsi_event *evt;
2644                 struct list_head *this, *tmp;
2645                 unsigned long flags;
2646
2647                 spin_lock_irqsave(&sdev->list_lock, flags);
2648                 list_splice_init(&sdev->event_list, &event_list);
2649                 spin_unlock_irqrestore(&sdev->list_lock, flags);
2650
2651                 if (list_empty(&event_list))
2652                         break;
2653
2654                 list_for_each_safe(this, tmp, &event_list) {
2655                         evt = list_entry(this, struct scsi_event, node);
2656                         list_del(&evt->node);
2657                         scsi_evt_emit(sdev, evt);
2658                         kfree(evt);
2659                 }
2660         }
2661 }
2662
2663 /**
2664  *      sdev_evt_send - send asserted event to uevent thread
2665  *      @sdev: scsi_device event occurred on
2666  *      @evt: event to send
2667  *
2668  *      Assert scsi device event asynchronously.
2669  */
2670 void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
2671 {
2672         unsigned long flags;
2673
2674 #if 0
2675         /* FIXME: currently this check eliminates all media change events
2676          * for polled devices.  Need to update to discriminate between AN
2677          * and polled events */
2678         if (!test_bit(evt->evt_type, sdev->supported_events)) {
2679                 kfree(evt);
2680                 return;
2681         }
2682 #endif
2683
2684         spin_lock_irqsave(&sdev->list_lock, flags);
2685         list_add_tail(&evt->node, &sdev->event_list);
2686         schedule_work(&sdev->event_work);
2687         spin_unlock_irqrestore(&sdev->list_lock, flags);
2688 }
2689 EXPORT_SYMBOL_GPL(sdev_evt_send);
2690
2691 /**
2692  *      sdev_evt_alloc - allocate a new scsi event
2693  *      @evt_type: type of event to allocate
2694  *      @gfpflags: GFP flags for allocation
2695  *
2696  *      Allocates and returns a new scsi_event.
2697  */
2698 struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,
2699                                   gfp_t gfpflags)
2700 {
2701         struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);
2702         if (!evt)
2703                 return NULL;
2704
2705         evt->evt_type = evt_type;
2706         INIT_LIST_HEAD(&evt->node);
2707
2708         /* evt_type-specific initialization, if any */
2709         switch (evt_type) {
2710         case SDEV_EVT_MEDIA_CHANGE:
2711         case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
2712         case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
2713         case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
2714         case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
2715         case SDEV_EVT_LUN_CHANGE_REPORTED:
2716         default:
2717                 /* do nothing */
2718                 break;
2719         }
2720
2721         return evt;
2722 }
2723 EXPORT_SYMBOL_GPL(sdev_evt_alloc);
2724
2725 /**
2726  *      sdev_evt_send_simple - send asserted event to uevent thread
2727  *      @sdev: scsi_device event occurred on
2728  *      @evt_type: type of event to send
2729  *      @gfpflags: GFP flags for allocation
2730  *
2731  *      Assert scsi device event asynchronously, given an event type.
2732  */
2733 void sdev_evt_send_simple(struct scsi_device *sdev,
2734                           enum scsi_device_event evt_type, gfp_t gfpflags)
2735 {
2736         struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);
2737         if (!evt) {
2738                 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",
2739                             evt_type);
2740                 return;
2741         }
2742
2743         sdev_evt_send(sdev, evt);
2744 }
2745 EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
2746
2747 /**
2748  *      scsi_device_quiesce - Block user issued commands.
2749  *      @sdev:  scsi device to quiesce.
2750  *
2751  *      This works by trying to transition to the SDEV_QUIESCE state
2752  *      (which must be a legal transition).  When the device is in this
2753  *      state, only special requests will be accepted, all others will
2754  *      be deferred.  Since special requests may also be requeued requests,
2755  *      a successful return doesn't guarantee the device will be 
2756  *      totally quiescent.
2757  *
2758  *      Must be called with user context, may sleep.
2759  *
2760  *      Returns zero if unsuccessful or an error if not.
2761  */
2762 int
2763 scsi_device_quiesce(struct scsi_device *sdev)
2764 {
2765         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2766         if (err)
2767                 return err;
2768
2769         scsi_run_queue(sdev->request_queue);
2770         while (atomic_read(&sdev->device_busy)) {
2771                 msleep_interruptible(200);
2772                 scsi_run_queue(sdev->request_queue);
2773         }
2774         return 0;
2775 }
2776 EXPORT_SYMBOL(scsi_device_quiesce);
2777
2778 /**
2779  *      scsi_device_resume - Restart user issued commands to a quiesced device.
2780  *      @sdev:  scsi device to resume.
2781  *
2782  *      Moves the device from quiesced back to running and restarts the
2783  *      queues.
2784  *
2785  *      Must be called with user context, may sleep.
2786  */
2787 void scsi_device_resume(struct scsi_device *sdev)
2788 {
2789         /* check if the device state was mutated prior to resume, and if
2790          * so assume the state is being managed elsewhere (for example
2791          * device deleted during suspend)
2792          */
2793         if (sdev->sdev_state != SDEV_QUIESCE ||
2794             scsi_device_set_state(sdev, SDEV_RUNNING))
2795                 return;
2796         scsi_run_queue(sdev->request_queue);
2797 }
2798 EXPORT_SYMBOL(scsi_device_resume);
2799
2800 static void
2801 device_quiesce_fn(struct scsi_device *sdev, void *data)
2802 {
2803         scsi_device_quiesce(sdev);
2804 }
2805
2806 void
2807 scsi_target_quiesce(struct scsi_target *starget)
2808 {
2809         starget_for_each_device(starget, NULL, device_quiesce_fn);
2810 }
2811 EXPORT_SYMBOL(scsi_target_quiesce);
2812
2813 static void
2814 device_resume_fn(struct scsi_device *sdev, void *data)
2815 {
2816         scsi_device_resume(sdev);
2817 }
2818
2819 void
2820 scsi_target_resume(struct scsi_target *starget)
2821 {
2822         starget_for_each_device(starget, NULL, device_resume_fn);
2823 }
2824 EXPORT_SYMBOL(scsi_target_resume);
2825
2826 /**
2827  * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state
2828  * @sdev:       device to block
2829  *
2830  * Block request made by scsi lld's to temporarily stop all
2831  * scsi commands on the specified device.  Called from interrupt
2832  * or normal process context.
2833  *
2834  * Returns zero if successful or error if not
2835  *
2836  * Notes:       
2837  *      This routine transitions the device to the SDEV_BLOCK state
2838  *      (which must be a legal transition).  When the device is in this
2839  *      state, all commands are deferred until the scsi lld reenables
2840  *      the device with scsi_device_unblock or device_block_tmo fires.
2841  */
2842 int
2843 scsi_internal_device_block(struct scsi_device *sdev)
2844 {
2845         struct request_queue *q = sdev->request_queue;
2846         unsigned long flags;
2847         int err = 0;
2848
2849         err = scsi_device_set_state(sdev, SDEV_BLOCK);
2850         if (err) {
2851                 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);
2852
2853                 if (err)
2854                         return err;
2855         }
2856
2857         /* 
2858          * The device has transitioned to SDEV_BLOCK.  Stop the
2859          * block layer from calling the midlayer with this device's
2860          * request queue. 
2861          */
2862         if (q->mq_ops) {
2863                 blk_mq_stop_hw_queues(q);
2864         } else {
2865                 spin_lock_irqsave(q->queue_lock, flags);
2866                 blk_stop_queue(q);
2867                 spin_unlock_irqrestore(q->queue_lock, flags);
2868         }
2869
2870         return 0;
2871 }
2872 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2873  
2874 /**
2875  * scsi_internal_device_unblock - resume a device after a block request
2876  * @sdev:       device to resume
2877  * @new_state:  state to set devices to after unblocking
2878  *
2879  * Called by scsi lld's or the midlayer to restart the device queue
2880  * for the previously suspended scsi device.  Called from interrupt or
2881  * normal process context.
2882  *
2883  * Returns zero if successful or error if not.
2884  *
2885  * Notes:       
2886  *      This routine transitions the device to the SDEV_RUNNING state
2887  *      or to one of the offline states (which must be a legal transition)
2888  *      allowing the midlayer to goose the queue for this device.
2889  */
2890 int
2891 scsi_internal_device_unblock(struct scsi_device *sdev,
2892                              enum scsi_device_state new_state)
2893 {
2894         struct request_queue *q = sdev->request_queue; 
2895         unsigned long flags;
2896
2897         /*
2898          * Try to transition the scsi device to SDEV_RUNNING or one of the
2899          * offlined states and goose the device queue if successful.
2900          */
2901         if ((sdev->sdev_state == SDEV_BLOCK) ||
2902             (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE))
2903                 sdev->sdev_state = new_state;
2904         else if (sdev->sdev_state == SDEV_CREATED_BLOCK) {
2905                 if (new_state == SDEV_TRANSPORT_OFFLINE ||
2906                     new_state == SDEV_OFFLINE)
2907                         sdev->sdev_state = new_state;
2908                 else
2909                         sdev->sdev_state = SDEV_CREATED;
2910         } else if (sdev->sdev_state != SDEV_CANCEL &&
2911                  sdev->sdev_state != SDEV_OFFLINE)
2912                 return -EINVAL;
2913
2914         if (q->mq_ops) {
2915                 blk_mq_start_stopped_hw_queues(q, false);
2916         } else {
2917                 spin_lock_irqsave(q->queue_lock, flags);
2918                 blk_start_queue(q);
2919                 spin_unlock_irqrestore(q->queue_lock, flags);
2920         }
2921
2922         return 0;
2923 }
2924 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2925
2926 static void
2927 device_block(struct scsi_device *sdev, void *data)
2928 {
2929         scsi_internal_device_block(sdev);
2930 }
2931
2932 static int
2933 target_block(struct device *dev, void *data)
2934 {
2935         if (scsi_is_target_device(dev))
2936                 starget_for_each_device(to_scsi_target(dev), NULL,
2937                                         device_block);
2938         return 0;
2939 }
2940
2941 void
2942 scsi_target_block(struct device *dev)
2943 {
2944         if (scsi_is_target_device(dev))
2945                 starget_for_each_device(to_scsi_target(dev), NULL,
2946                                         device_block);
2947         else
2948                 device_for_each_child(dev, NULL, target_block);
2949 }
2950 EXPORT_SYMBOL_GPL(scsi_target_block);
2951
2952 static void
2953 device_unblock(struct scsi_device *sdev, void *data)
2954 {
2955         scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data);
2956 }
2957
2958 static int
2959 target_unblock(struct device *dev, void *data)
2960 {
2961         if (scsi_is_target_device(dev))
2962                 starget_for_each_device(to_scsi_target(dev), data,
2963                                         device_unblock);
2964         return 0;
2965 }
2966
2967 void
2968 scsi_target_unblock(struct device *dev, enum scsi_device_state new_state)
2969 {
2970         if (scsi_is_target_device(dev))
2971                 starget_for_each_device(to_scsi_target(dev), &new_state,
2972                                         device_unblock);
2973         else
2974                 device_for_each_child(dev, &new_state, target_unblock);
2975 }
2976 EXPORT_SYMBOL_GPL(scsi_target_unblock);
2977
2978 /**
2979  * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
2980  * @sgl:        scatter-gather list
2981  * @sg_count:   number of segments in sg
2982  * @offset:     offset in bytes into sg, on return offset into the mapped area
2983  * @len:        bytes to map, on return number of bytes mapped
2984  *
2985  * Returns virtual address of the start of the mapped page
2986  */
2987 void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
2988                           size_t *offset, size_t *len)
2989 {
2990         int i;
2991         size_t sg_len = 0, len_complete = 0;
2992         struct scatterlist *sg;
2993         struct page *page;
2994
2995         WARN_ON(!irqs_disabled());
2996
2997         for_each_sg(sgl, sg, sg_count, i) {
2998                 len_complete = sg_len; /* Complete sg-entries */
2999                 sg_len += sg->length;
3000                 if (sg_len > *offset)
3001                         break;
3002         }
3003
3004         if (unlikely(i == sg_count)) {
3005                 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
3006                         "elements %d\n",
3007                        __func__, sg_len, *offset, sg_count);
3008                 WARN_ON(1);
3009                 return NULL;
3010         }
3011
3012         /* Offset starting from the beginning of first page in this sg-entry */
3013         *offset = *offset - len_complete + sg->offset;
3014
3015         /* Assumption: contiguous pages can be accessed as "page + i" */
3016         page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));
3017         *offset &= ~PAGE_MASK;
3018
3019         /* Bytes in this sg-entry from *offset to the end of the page */
3020         sg_len = PAGE_SIZE - *offset;
3021         if (*len > sg_len)
3022                 *len = sg_len;
3023
3024         return kmap_atomic(page);
3025 }
3026 EXPORT_SYMBOL(scsi_kmap_atomic_sg);
3027
3028 /**
3029  * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg
3030  * @virt:       virtual address to be unmapped
3031  */
3032 void scsi_kunmap_atomic_sg(void *virt)
3033 {
3034         kunmap_atomic(virt);
3035 }
3036 EXPORT_SYMBOL(scsi_kunmap_atomic_sg);
3037
3038 void sdev_disable_disk_events(struct scsi_device *sdev)
3039 {
3040         atomic_inc(&sdev->disk_events_disable_depth);
3041 }
3042 EXPORT_SYMBOL(sdev_disable_disk_events);
3043
3044 void sdev_enable_disk_events(struct scsi_device *sdev)
3045 {
3046         if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0))
3047                 return;
3048         atomic_dec(&sdev->disk_events_disable_depth);
3049 }
3050 EXPORT_SYMBOL(sdev_enable_disk_events);