drivers/dma/dmatest.c

   1 /*
   2  * DMA Engine test module
   3  *
   4  * Copyright (C) 2007 Atmel Corporation
   5  * Copyright (C) 2013 Intel Corporation
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/freezer.h>
  15 #include <linux/init.h>
  16 #include <linux/kthread.h>
  17 #include <linux/module.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/random.h>
  20 #include <linux/slab.h>
  21 #include <linux/wait.h>
  22 #include <linux/ctype.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/seq_file.h>
  26
  27 static unsigned int test_buf_size = 16384;
  28 module_param(test_buf_size, uint, S_IRUGO);
  29 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  30
  31 static char test_channel[20];
  32 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
  33 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  34
  35 static char test_device[20];
  36 module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
  37 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  38
  39 static unsigned int threads_per_chan = 1;
  40 module_param(threads_per_chan, uint, S_IRUGO);
  41 MODULE_PARM_DESC(threads_per_chan,
  42                 "Number of threads to start per channel (default: 1)");
  43
  44 static unsigned int max_channels;
  45 module_param(max_channels, uint, S_IRUGO);
  46 MODULE_PARM_DESC(max_channels,
  47                 "Maximum number of channels to use (default: all)");
  48
  49 static unsigned int iterations;
  50 module_param(iterations, uint, S_IRUGO);
  51 MODULE_PARM_DESC(iterations,
  52                 "Iterations before stopping test (default: infinite)");
  53
  54 static unsigned int xor_sources = 3;
  55 module_param(xor_sources, uint, S_IRUGO);
  56 MODULE_PARM_DESC(xor_sources,
  57                 "Number of xor source buffers (default: 3)");
  58
  59 static unsigned int pq_sources = 3;
  60 module_param(pq_sources, uint, S_IRUGO);
  61 MODULE_PARM_DESC(pq_sources,
  62                 "Number of p+q source buffers (default: 3)");
  63
  64 static int timeout = 3000;
  65 module_param(timeout, uint, S_IRUGO);
  66 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  67                  "Pass -1 for infinite timeout");
  68
  69 /* Maximum amount of mismatched bytes in buffer to print */
  70 #define MAX_ERROR_COUNT         32
  71
  72 /*
  73  * Initialization patterns. All bytes in the source buffer has bit 7
  74  * set, all bytes in the destination buffer has bit 7 cleared.
  75  *
  76  * Bit 6 is set for all bytes which are to be copied by the DMA
  77  * engine. Bit 5 is set for all bytes which are to be overwritten by
  78  * the DMA engine.
  79  *
  80  * The remaining bits are the inverse of a counter which increments by
  81  * one for each byte address.
  82  */
  83 #define PATTERN_SRC             0x80
  84 #define PATTERN_DST             0x00
  85 #define PATTERN_COPY            0x40
  86 #define PATTERN_OVERWRITE       0x20
  87 #define PATTERN_COUNT_MASK      0x1f
  88
  89 enum dmatest_error_type {
  90         DMATEST_ET_OK,
  91         DMATEST_ET_MAP_SRC,
  92         DMATEST_ET_MAP_DST,
  93         DMATEST_ET_PREP,
  94         DMATEST_ET_SUBMIT,
  95         DMATEST_ET_TIMEOUT,
  96         DMATEST_ET_DMA_ERROR,
  97         DMATEST_ET_DMA_IN_PROGRESS,
  98         DMATEST_ET_VERIFY,
  99         DMATEST_ET_VERIFY_BUF,
 100 };
 101
 102 struct dmatest_verify_buffer {
 103         unsigned int    index;
 104         u8              expected;
 105         u8              actual;
 106 };
 107
 108 struct dmatest_verify_result {
 109         unsigned int                    error_count;
 110         struct dmatest_verify_buffer    data[MAX_ERROR_COUNT];
 111         u8                              pattern;
 112         bool                            is_srcbuf;
 113 };
 114
 115 struct dmatest_thread_result {
 116         struct list_head        node;
 117         unsigned int            n;
 118         unsigned int            src_off;
 119         unsigned int            dst_off;
 120         unsigned int            len;
 121         enum dmatest_error_type type;
 122         union {
 123                 unsigned long                   data;
 124                 dma_cookie_t                    cookie;
 125                 enum dma_status                 status;
 126                 int                             error;
 127                 struct dmatest_verify_result    *vr;
 128         };
 129 };
 130
 131 struct dmatest_result {
 132         struct list_head        node;
 133         char                    *name;
 134         struct list_head        results;
 135 };
 136
 137 struct dmatest_info;
 138
 139 struct dmatest_thread {
 140         struct list_head        node;
 141         struct dmatest_info     *info;
 142         struct task_struct      *task;
 143         struct dma_chan         *chan;
 144         u8                      **srcs;
 145         u8                      **dsts;
 146         enum dma_transaction_type type;
 147         bool                    done;
 148 };
 149
 150 struct dmatest_chan {
 151         struct list_head        node;
 152         struct dma_chan         *chan;
 153         struct list_head        threads;
 154 };
 155
 156 /**
 157  * struct dmatest_params - test parameters.
 158  * @buf_size:           size of the memcpy test buffer
 159  * @channel:            bus ID of the channel to test
 160  * @device:             bus ID of the DMA Engine to test
 161  * @threads_per_chan:   number of threads to start per channel
 162  * @max_channels:       maximum number of channels to use
 163  * @iterations:         iterations before stopping test
 164  * @xor_sources:        number of xor source buffers
 165  * @pq_sources:         number of p+q source buffers
 166  * @timeout:            transfer timeout in msec, -1 for infinite timeout
 167  */
 168 struct dmatest_params {
 169         unsigned int    buf_size;
 170         char            channel[20];
 171         char            device[20];
 172         unsigned int    threads_per_chan;
 173         unsigned int    max_channels;
 174         unsigned int    iterations;
 175         unsigned int    xor_sources;
 176         unsigned int    pq_sources;
 177         int             timeout;
 178 };
 179
 180 /**
 181  * struct dmatest_info - test information.
 182  * @params:             test parameters
 183  * @lock:               access protection to the fields of this structure
 184  */
 185 struct dmatest_info {
 186         /* Test parameters */
 187         struct dmatest_params   params;
 188
 189         /* Internal state */
 190         struct list_head        channels;
 191         unsigned int            nr_channels;
 192         struct mutex            lock;
 193
 194         /* debugfs related stuff */
 195         struct dentry           *root;
 196         struct dmatest_params   dbgfs_params;
 197
 198         /* Test results */
 199         struct list_head        results;
 200         struct mutex            results_lock;
 201 };
 202
 203 static struct dmatest_info test_info;
 204
 205 static bool dmatest_match_channel(struct dmatest_params *params,
 206                 struct dma_chan *chan)
 207 {
 208         if (params->channel[0] == '\0')
 209                 return true;
 210         return strcmp(dma_chan_name(chan), params->channel) == 0;
 211 }
 212
 213 static bool dmatest_match_device(struct dmatest_params *params,
 214                 struct dma_device *device)
 215 {
 216         if (params->device[0] == '\0')
 217                 return true;
 218         return strcmp(dev_name(device->dev), params->device) == 0;
 219 }
 220
 221 static unsigned long dmatest_random(void)
 222 {
 223         unsigned long buf;
 224
 225         get_random_bytes(&buf, sizeof(buf));
 226         return buf;
 227 }
 228
 229 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 230                 unsigned int buf_size)
 231 {
 232         unsigned int i;
 233         u8 *buf;
 234
 235         for (; (buf = *bufs); bufs++) {
 236                 for (i = 0; i < start; i++)
 237                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 238                 for ( ; i < start + len; i++)
 239                         buf[i] = PATTERN_SRC | PATTERN_COPY
 240                                 | (~i & PATTERN_COUNT_MASK);
 241                 for ( ; i < buf_size; i++)
 242                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 243                 buf++;
 244         }
 245 }
 246
 247 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 248                 unsigned int buf_size)
 249 {
 250         unsigned int i;
 251         u8 *buf;
 252
 253         for (; (buf = *bufs); bufs++) {
 254                 for (i = 0; i < start; i++)
 255                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 256                 for ( ; i < start + len; i++)
 257                         buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 258                                 | (~i & PATTERN_COUNT_MASK);
 259                 for ( ; i < buf_size; i++)
 260                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 261         }
 262 }
 263
 264 static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
 265                 unsigned int start, unsigned int end, unsigned int counter,
 266                 u8 pattern, bool is_srcbuf)
 267 {
 268         unsigned int i;
 269         unsigned int error_count = 0;
 270         u8 actual;
 271         u8 expected;
 272         u8 *buf;
 273         unsigned int counter_orig = counter;
 274         struct dmatest_verify_buffer *vb;
 275
 276         for (; (buf = *bufs); bufs++) {
 277                 counter = counter_orig;
 278                 for (i = start; i < end; i++) {
 279                         actual = buf[i];
 280                         expected = pattern | (~counter & PATTERN_COUNT_MASK);
 281                         if (actual != expected) {
 282                                 if (error_count < MAX_ERROR_COUNT && vr) {
 283                                         vb = &vr->data[error_count];
 284                                         vb->index = i;
 285                                         vb->expected = expected;
 286                                         vb->actual = actual;
 287                                 }
 288                                 error_count++;
 289                         }
 290                         counter++;
 291                 }
 292         }
 293
 294         if (error_count > MAX_ERROR_COUNT)
 295                 pr_warning("%s: %u errors suppressed\n",
 296                         current->comm, error_count - MAX_ERROR_COUNT);
 297
 298         return error_count;
 299 }
 300
 301 /* poor man's completion - we want to use wait_event_freezable() on it */
 302 struct dmatest_done {
 303         bool                    done;
 304         wait_queue_head_t       *wait;
 305 };
 306
 307 static void dmatest_callback(void *arg)
 308 {
 309         struct dmatest_done *done = arg;
 310
 311         done->done = true;
 312         wake_up_all(done->wait);
 313 }
 314
 315 static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
 316                              unsigned int count)
 317 {
 318         while (count--)
 319                 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
 320 }
 321
 322 static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
 323                              unsigned int count)
 324 {
 325         while (count--)
 326                 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
 327 }
 328
 329 static unsigned int min_odd(unsigned int x, unsigned int y)
 330 {
 331         unsigned int val = min(x, y);
 332
 333         return val % 2 ? val : val - 1;
 334 }
 335
 336 static char *verify_result_get_one(struct dmatest_verify_result *vr,
 337                 unsigned int i)
 338 {
 339         struct dmatest_verify_buffer *vb = &vr->data[i];
 340         u8 diff = vb->actual ^ vr->pattern;
 341         static char buf[512];
 342         char *msg;
 343
 344         if (vr->is_srcbuf)
 345                 msg = "srcbuf overwritten!";
 346         else if ((vr->pattern & PATTERN_COPY)
 347                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 348                 msg = "dstbuf not copied!";
 349         else if (diff & PATTERN_SRC)
 350                 msg = "dstbuf was copied!";
 351         else
 352                 msg = "dstbuf mismatch!";
 353
 354         snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
 355                  vb->index, vb->expected, vb->actual);
 356
 357         return buf;
 358 }
 359
 360 static char *thread_result_get(const char *name,
 361                 struct dmatest_thread_result *tr)
 362 {
 363         static const char * const messages[] = {
 364                 [DMATEST_ET_OK]                 = "No errors",
 365                 [DMATEST_ET_MAP_SRC]            = "src mapping error",
 366                 [DMATEST_ET_MAP_DST]            = "dst mapping error",
 367                 [DMATEST_ET_PREP]               = "prep error",
 368                 [DMATEST_ET_SUBMIT]             = "submit error",
 369                 [DMATEST_ET_TIMEOUT]            = "test timed out",
 370                 [DMATEST_ET_DMA_ERROR]          =
 371                         "got completion callback (DMA_ERROR)",
 372                 [DMATEST_ET_DMA_IN_PROGRESS]    =
 373                         "got completion callback (DMA_IN_PROGRESS)",
 374                 [DMATEST_ET_VERIFY]             = "errors",
 375                 [DMATEST_ET_VERIFY_BUF]         = "verify errors",
 376         };
 377         static char buf[512];
 378
 379         snprintf(buf, sizeof(buf) - 1,
 380                  "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 381                  name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
 382                  tr->len, tr->data);
 383
 384         return buf;
 385 }
 386
 387 static int thread_result_add(struct dmatest_info *info,
 388                 struct dmatest_result *r, enum dmatest_error_type type,
 389                 unsigned int n, unsigned int src_off, unsigned int dst_off,
 390                 unsigned int len, unsigned long data)
 391 {
 392         struct dmatest_thread_result *tr;
 393
 394         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 395         if (!tr)
 396                 return -ENOMEM;
 397
 398         tr->type = type;
 399         tr->n = n;
 400         tr->src_off = src_off;
 401         tr->dst_off = dst_off;
 402         tr->len = len;
 403         tr->data = data;
 404
 405         mutex_lock(&info->results_lock);
 406         list_add_tail(&tr->node, &r->results);
 407         mutex_unlock(&info->results_lock);
 408
 409         pr_warn("%s\n", thread_result_get(r->name, tr));
 410         return 0;
 411 }
 412
 413 static unsigned int verify_result_add(struct dmatest_info *info,
 414                 struct dmatest_result *r, unsigned int n,
 415                 unsigned int src_off, unsigned int dst_off, unsigned int len,
 416                 u8 **bufs, int whence, unsigned int counter, u8 pattern,
 417                 bool is_srcbuf)
 418 {
 419         struct dmatest_verify_result *vr;
 420         unsigned int error_count;
 421         unsigned int buf_off = is_srcbuf ? src_off : dst_off;
 422         unsigned int start, end;
 423
 424         if (whence < 0) {
 425                 start = 0;
 426                 end = buf_off;
 427         } else if (whence > 0) {
 428                 start = buf_off + len;
 429                 end = info->params.buf_size;
 430         } else {
 431                 start = buf_off;
 432                 end = buf_off + len;
 433         }
 434
 435         vr = kmalloc(sizeof(*vr), GFP_KERNEL);
 436         if (!vr) {
 437                 pr_warn("dmatest: No memory to store verify result\n");
 438                 return dmatest_verify(NULL, bufs, start, end, counter, pattern,
 439                                       is_srcbuf);
 440         }
 441
 442         vr->pattern = pattern;
 443         vr->is_srcbuf = is_srcbuf;
 444
 445         error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
 446                                      is_srcbuf);
 447         if (error_count) {
 448                 vr->error_count = error_count;
 449                 thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
 450                                   dst_off, len, (unsigned long)vr);
 451                 return error_count;
 452         }
 453
 454         kfree(vr);
 455         return 0;
 456 }
 457
 458 static void result_free(struct dmatest_info *info, const char *name)
 459 {
 460         struct dmatest_result *r, *_r;
 461
 462         mutex_lock(&info->results_lock);
 463         list_for_each_entry_safe(r, _r, &info->results, node) {
 464                 struct dmatest_thread_result *tr, *_tr;
 465
 466                 if (name && strcmp(r->name, name))
 467                         continue;
 468
 469                 list_for_each_entry_safe(tr, _tr, &r->results, node) {
 470                         if (tr->type == DMATEST_ET_VERIFY_BUF)
 471                                 kfree(tr->vr);
 472                         list_del(&tr->node);
 473                         kfree(tr);
 474                 }
 475
 476                 kfree(r->name);
 477                 list_del(&r->node);
 478                 kfree(r);
 479         }
 480
 481         mutex_unlock(&info->results_lock);
 482 }
 483
 484 static struct dmatest_result *result_init(struct dmatest_info *info,
 485                 const char *name)
 486 {
 487         struct dmatest_result *r;
 488
 489         r = kzalloc(sizeof(*r), GFP_KERNEL);
 490         if (r) {
 491                 r->name = kstrdup(name, GFP_KERNEL);
 492                 INIT_LIST_HEAD(&r->results);
 493                 mutex_lock(&info->results_lock);
 494                 list_add_tail(&r->node, &info->results);
 495                 mutex_unlock(&info->results_lock);
 496         }
 497         return r;
 498 }
 499
 500 /*
 501  * This function repeatedly tests DMA transfers of various lengths and
 502  * offsets for a given operation type until it is told to exit by
 503  * kthread_stop(). There may be multiple threads running this function
 504  * in parallel for a single channel, and there may be multiple channels
 505  * being tested in parallel.
 506  *
 507  * Before each test, the source and destination buffer is initialized
 508  * with a known pattern. This pattern is different depending on
 509  * whether it's in an area which is supposed to be copied or
 510  * overwritten, and different in the source and destination buffers.
 511  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 512  * we'll notice.
 513  */
 514 static int dmatest_func(void *data)
 515 {
 516         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 517         struct dmatest_thread   *thread = data;
 518         struct dmatest_done     done = { .wait = &done_wait };
 519         struct dmatest_info     *info;
 520         struct dmatest_params   *params;
 521         struct dma_chan         *chan;
 522         struct dma_device       *dev;
 523         const char              *thread_name;
 524         unsigned int            src_off, dst_off, len;
 525         unsigned int            error_count;
 526         unsigned int            failed_tests = 0;
 527         unsigned int            total_tests = 0;
 528         dma_cookie_t            cookie;
 529         enum dma_status         status;
 530         enum dma_ctrl_flags     flags;
 531         u8                      *pq_coefs = NULL;
 532         int                     ret;
 533         int                     src_cnt;
 534         int                     dst_cnt;
 535         int                     i;
 536         struct dmatest_result   *result;
 537
 538         thread_name = current->comm;
 539         set_freezable();
 540
 541         ret = -ENOMEM;
 542
 543         smp_rmb();
 544         info = thread->info;
 545         params = &info->params;
 546         chan = thread->chan;
 547         dev = chan->device;
 548         if (thread->type == DMA_MEMCPY)
 549                 src_cnt = dst_cnt = 1;
 550         else if (thread->type == DMA_XOR) {
 551                 /* force odd to ensure dst = src */
 552                 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 553                 dst_cnt = 1;
 554         } else if (thread->type == DMA_PQ) {
 555                 /* force odd to ensure dst = src */
 556                 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 557                 dst_cnt = 2;
 558
 559                 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 560                 if (!pq_coefs)
 561                         goto err_thread_type;
 562
 563                 for (i = 0; i < src_cnt; i++)
 564                         pq_coefs[i] = 1;
 565         } else
 566                 goto err_thread_type;
 567
 568         result = result_init(info, thread_name);
 569         if (!result)
 570                 goto err_srcs;
 571
 572         thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 573         if (!thread->srcs)
 574                 goto err_srcs;
 575         for (i = 0; i < src_cnt; i++) {
 576                 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 577                 if (!thread->srcs[i])
 578                         goto err_srcbuf;
 579         }
 580         thread->srcs[i] = NULL;
 581
 582         thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 583         if (!thread->dsts)
 584                 goto err_dsts;
 585         for (i = 0; i < dst_cnt; i++) {
 586                 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 587                 if (!thread->dsts[i])
 588                         goto err_dstbuf;
 589         }
 590         thread->dsts[i] = NULL;
 591
 592         set_user_nice(current, 10);
 593
 594         /*
 595          * src buffers are freed by the DMAEngine code with dma_unmap_single()
 596          * dst buffers are freed by ourselves below
 597          */
 598         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
 599               | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
 600
 601         while (!kthread_should_stop()
 602                && !(params->iterations && total_tests >= params->iterations)) {
 603                 struct dma_async_tx_descriptor *tx = NULL;
 604                 dma_addr_t dma_srcs[src_cnt];
 605                 dma_addr_t dma_dsts[dst_cnt];
 606                 u8 align = 0;
 607
 608                 total_tests++;
 609
 610                 /* honor alignment restrictions */
 611                 if (thread->type == DMA_MEMCPY)
 612                         align = dev->copy_align;
 613                 else if (thread->type == DMA_XOR)
 614                         align = dev->xor_align;
 615                 else if (thread->type == DMA_PQ)
 616                         align = dev->pq_align;
 617
 618                 if (1 << align > params->buf_size) {
 619                         pr_err("%u-byte buffer too small for %d-byte alignment\n",
 620                                params->buf_size, 1 << align);
 621                         break;
 622                 }
 623
 624                 len = dmatest_random() % params->buf_size + 1;
 625                 len = (len >> align) << align;
 626                 if (!len)
 627                         len = 1 << align;
 628                 src_off = dmatest_random() % (params->buf_size - len + 1);
 629                 dst_off = dmatest_random() % (params->buf_size - len + 1);
 630
 631                 src_off = (src_off >> align) << align;
 632                 dst_off = (dst_off >> align) << align;
 633
 634                 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size);
 635                 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size);
 636
 637                 for (i = 0; i < src_cnt; i++) {
 638                         u8 *buf = thread->srcs[i] + src_off;
 639
 640                         dma_srcs[i] = dma_map_single(dev->dev, buf, len,
 641                                                      DMA_TO_DEVICE);
 642                         ret = dma_mapping_error(dev->dev, dma_srcs[i]);
 643                         if (ret) {
 644                                 unmap_src(dev->dev, dma_srcs, len, i);
 645                                 thread_result_add(info, result,
 646                                                   DMATEST_ET_MAP_SRC,
 647                                                   total_tests, src_off, dst_off,
 648                                                   len, ret);
 649                                 failed_tests++;
 650                                 continue;
 651                         }
 652                 }
 653                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 654                 for (i = 0; i < dst_cnt; i++) {
 655                         dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
 656                                                      params->buf_size,
 657                                                      DMA_BIDIRECTIONAL);
 658                         ret = dma_mapping_error(dev->dev, dma_dsts[i]);
 659                         if (ret) {
 660                                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 661                                 unmap_dst(dev->dev, dma_dsts, params->buf_size,
 662                                           i);
 663                                 thread_result_add(info, result,
 664                                                   DMATEST_ET_MAP_DST,
 665                                                   total_tests, src_off, dst_off,
 666                                                   len, ret);
 667                                 failed_tests++;
 668                                 continue;
 669                         }
 670                 }
 671
 672                 if (thread->type == DMA_MEMCPY)
 673                         tx = dev->device_prep_dma_memcpy(chan,
 674                                                          dma_dsts[0] + dst_off,
 675                                                          dma_srcs[0], len,
 676                                                          flags);
 677                 else if (thread->type == DMA_XOR)
 678                         tx = dev->device_prep_dma_xor(chan,
 679                                                       dma_dsts[0] + dst_off,
 680                                                       dma_srcs, src_cnt,
 681                                                       len, flags);
 682                 else if (thread->type == DMA_PQ) {
 683                         dma_addr_t dma_pq[dst_cnt];
 684
 685                         for (i = 0; i < dst_cnt; i++)
 686                                 dma_pq[i] = dma_dsts[i] + dst_off;
 687                         tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
 688                                                      src_cnt, pq_coefs,
 689                                                      len, flags);
 690                 }
 691
 692                 if (!tx) {
 693                         unmap_src(dev->dev, dma_srcs, len, src_cnt);
 694                         unmap_dst(dev->dev, dma_dsts, params->buf_size,
 695                                   dst_cnt);
 696                         thread_result_add(info, result, DMATEST_ET_PREP,
 697                                           total_tests, src_off, dst_off,
 698                                           len, 0);
 699                         msleep(100);
 700                         failed_tests++;
 701                         continue;
 702                 }
 703
 704                 done.done = false;
 705                 tx->callback = dmatest_callback;
 706                 tx->callback_param = &done;
 707                 cookie = tx->tx_submit(tx);
 708
 709                 if (dma_submit_error(cookie)) {
 710                         thread_result_add(info, result, DMATEST_ET_SUBMIT,
 711                                           total_tests, src_off, dst_off,
 712                                           len, cookie);
 713                         msleep(100);
 714                         failed_tests++;
 715                         continue;
 716                 }
 717                 dma_async_issue_pending(chan);
 718
 719                 wait_event_freezable_timeout(done_wait,
 720                                              done.done || kthread_should_stop(),
 721                                              msecs_to_jiffies(params->timeout));
 722
 723                 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 724
 725                 if (!done.done) {
 726                         /*
 727                          * We're leaving the timed out dma operation with
 728                          * dangling pointer to done_wait.  To make this
 729                          * correct, we'll need to allocate wait_done for
 730                          * each test iteration and perform "who's gonna
 731                          * free it this time?" dancing.  For now, just
 732                          * leave it dangling.
 733                          */
 734                         thread_result_add(info, result, DMATEST_ET_TIMEOUT,
 735                                           total_tests, src_off, dst_off,
 736                                           len, 0);
 737                         failed_tests++;
 738                         continue;
 739                 } else if (status != DMA_SUCCESS) {
 740                         enum dmatest_error_type type = (status == DMA_ERROR) ?
 741                                 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS;
 742                         thread_result_add(info, result, type,
 743                                           total_tests, src_off, dst_off,
 744                                           len, status);
 745                         failed_tests++;
 746                         continue;
 747                 }
 748
 749                 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
 750                 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
 751
 752                 error_count = 0;
 753
 754                 pr_debug("%s: verifying source buffer...\n", thread_name);
 755                 error_count += verify_result_add(info, result, total_tests,
 756                                 src_off, dst_off, len, thread->srcs, -1,
 757                                 0, PATTERN_SRC, true);
 758                 error_count += verify_result_add(info, result, total_tests,
 759                                 src_off, dst_off, len, thread->srcs, 0,
 760                                 src_off, PATTERN_SRC | PATTERN_COPY, true);
 761                 error_count += verify_result_add(info, result, total_tests,
 762                                 src_off, dst_off, len, thread->srcs, 1,
 763                                 src_off + len, PATTERN_SRC, true);
 764
 765                 pr_debug("%s: verifying dest buffer...\n", thread_name);
 766                 error_count += verify_result_add(info, result, total_tests,
 767                                 src_off, dst_off, len, thread->dsts, -1,
 768                                 0, PATTERN_DST, false);
 769                 error_count += verify_result_add(info, result, total_tests,
 770                                 src_off, dst_off, len, thread->dsts, 0,
 771                                 src_off, PATTERN_SRC | PATTERN_COPY, false);
 772                 error_count += verify_result_add(info, result, total_tests,
 773                                 src_off, dst_off, len, thread->dsts, 1,
 774                                 dst_off + len, PATTERN_DST, false);
 775
 776                 if (error_count) {
 777                         thread_result_add(info, result, DMATEST_ET_VERIFY,
 778                                           total_tests, src_off, dst_off,
 779                                           len, error_count);
 780                         failed_tests++;
 781                 } else {
 782                         thread_result_add(info, result, DMATEST_ET_OK,
 783                                           total_tests, src_off, dst_off,
 784                                           len, 0);
 785                 }
 786         }
 787
 788         ret = 0;
 789         for (i = 0; thread->dsts[i]; i++)
 790                 kfree(thread->dsts[i]);
 791 err_dstbuf:
 792         kfree(thread->dsts);
 793 err_dsts:
 794         for (i = 0; thread->srcs[i]; i++)
 795                 kfree(thread->srcs[i]);
 796 err_srcbuf:
 797         kfree(thread->srcs);
 798 err_srcs:
 799         kfree(pq_coefs);
 800 err_thread_type:
 801         pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
 802                         thread_name, total_tests, failed_tests, ret);
 803
 804         /* terminate all transfers on specified channels */
 805         if (ret)
 806                 dmaengine_terminate_all(chan);
 807
 808         thread->done = true;
 809
 810         if (params->iterations > 0)
 811                 while (!kthread_should_stop()) {
 812                         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
 813                         interruptible_sleep_on(&wait_dmatest_exit);
 814                 }
 815
 816         return ret;
 817 }
 818
 819 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 820 {
 821         struct dmatest_thread   *thread;
 822         struct dmatest_thread   *_thread;
 823         int                     ret;
 824
 825         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 826                 ret = kthread_stop(thread->task);
 827                 pr_debug("dmatest: thread %s exited with status %d\n",
 828                                 thread->task->comm, ret);
 829                 list_del(&thread->node);
 830                 kfree(thread);
 831         }
 832
 833         /* terminate all transfers on specified channels */
 834         dmaengine_terminate_all(dtc->chan);
 835
 836         kfree(dtc);
 837 }
 838
 839 static int dmatest_add_threads(struct dmatest_info *info,
 840                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 841 {
 842         struct dmatest_params *params = &info->params;
 843         struct dmatest_thread *thread;
 844         struct dma_chan *chan = dtc->chan;
 845         char *op;
 846         unsigned int i;
 847
 848         if (type == DMA_MEMCPY)
 849                 op = "copy";
 850         else if (type == DMA_XOR)
 851                 op = "xor";
 852         else if (type == DMA_PQ)
 853                 op = "pq";
 854         else
 855                 return -EINVAL;
 856
 857         for (i = 0; i < params->threads_per_chan; i++) {
 858                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 859                 if (!thread) {
 860                         pr_warning("dmatest: No memory for %s-%s%u\n",
 861                                    dma_chan_name(chan), op, i);
 862
 863                         break;
 864                 }
 865                 thread->info = info;
 866                 thread->chan = dtc->chan;
 867                 thread->type = type;
 868                 smp_wmb();
 869                 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
 870                                 dma_chan_name(chan), op, i);
 871                 if (IS_ERR(thread->task)) {
 872                         pr_warning("dmatest: Failed to run thread %s-%s%u\n",
 873                                         dma_chan_name(chan), op, i);
 874                         kfree(thread);
 875                         break;
 876                 }
 877
 878                 /* srcbuf and dstbuf are allocated by the thread itself */
 879
 880                 list_add_tail(&thread->node, &dtc->threads);
 881         }
 882
 883         return i;
 884 }
 885
 886 static int dmatest_add_channel(struct dmatest_info *info,
 887                 struct dma_chan *chan)
 888 {
 889         struct dmatest_chan     *dtc;
 890         struct dma_device       *dma_dev = chan->device;
 891         unsigned int            thread_count = 0;
 892         int cnt;
 893
 894         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 895         if (!dtc) {
 896                 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
 897                 return -ENOMEM;
 898         }
 899
 900         dtc->chan = chan;
 901         INIT_LIST_HEAD(&dtc->threads);
 902
 903         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 904                 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 905                 thread_count += cnt > 0 ? cnt : 0;
 906         }
 907         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 908                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 909                 thread_count += cnt > 0 ? cnt : 0;
 910         }
 911         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 912                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 913                 thread_count += cnt > 0 ? cnt : 0;
 914         }
 915
 916         pr_info("dmatest: Started %u threads using %s\n",
 917                 thread_count, dma_chan_name(chan));
 918
 919         list_add_tail(&dtc->node, &info->channels);
 920         info->nr_channels++;
 921
 922         return 0;
 923 }
 924
 925 static bool filter(struct dma_chan *chan, void *param)
 926 {
 927         struct dmatest_params *params = param;
 928
 929         if (!dmatest_match_channel(params, chan) ||
 930             !dmatest_match_device(params, chan->device))
 931                 return false;
 932         else
 933                 return true;
 934 }
 935
 936 static int __run_threaded_test(struct dmatest_info *info)
 937 {
 938         dma_cap_mask_t mask;
 939         struct dma_chan *chan;
 940         struct dmatest_params *params = &info->params;
 941         int err = 0;
 942
 943         dma_cap_zero(mask);
 944         dma_cap_set(DMA_MEMCPY, mask);
 945         for (;;) {
 946                 chan = dma_request_channel(mask, filter, params);
 947                 if (chan) {
 948                         err = dmatest_add_channel(info, chan);
 949                         if (err) {
 950                                 dma_release_channel(chan);
 951                                 break; /* add_channel failed, punt */
 952                         }
 953                 } else
 954                         break; /* no more channels available */
 955                 if (params->max_channels &&
 956                     info->nr_channels >= params->max_channels)
 957                         break; /* we have all we need */
 958         }
 959         return err;
 960 }
 961
 962 #ifndef MODULE
 963 static int run_threaded_test(struct dmatest_info *info)
 964 {
 965         int ret;
 966
 967         mutex_lock(&info->lock);
 968         ret = __run_threaded_test(info);
 969         mutex_unlock(&info->lock);
 970         return ret;
 971 }
 972 #endif
 973
 974 static void __stop_threaded_test(struct dmatest_info *info)
 975 {
 976         struct dmatest_chan *dtc, *_dtc;
 977         struct dma_chan *chan;
 978
 979         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 980                 list_del(&dtc->node);
 981                 chan = dtc->chan;
 982                 dmatest_cleanup_channel(dtc);
 983                 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan));
 984                 dma_release_channel(chan);
 985         }
 986
 987         info->nr_channels = 0;
 988 }
 989
 990 static void stop_threaded_test(struct dmatest_info *info)
 991 {
 992         mutex_lock(&info->lock);
 993         __stop_threaded_test(info);
 994         mutex_unlock(&info->lock);
 995 }
 996
 997 static int __restart_threaded_test(struct dmatest_info *info, bool run)
 998 {
 999         struct dmatest_params *params = &info->params;
1000         int ret;
1001
1002         /* Stop any running test first */
1003         __stop_threaded_test(info);
1004
1005         if (run == false)
1006                 return 0;
1007
1008         /* Clear results from previous run */
1009         result_free(info, NULL);
1010
1011         /* Copy test parameters */
1012         memcpy(params, &info->dbgfs_params, sizeof(*params));
1013
1014         /* Run test with new parameters */
1015         ret = __run_threaded_test(info);
1016         if (ret) {
1017                 __stop_threaded_test(info);
1018                 pr_err("dmatest: Can't run test\n");
1019         }
1020
1021         return ret;
1022 }
1023
1024 static ssize_t dtf_write_string(void *to, size_t available, loff_t *ppos,
1025                 const void __user *from, size_t count)
1026 {
1027         char tmp[20];
1028         ssize_t len;
1029
1030         len = simple_write_to_buffer(tmp, sizeof(tmp) - 1, ppos, from, count);
1031         if (len >= 0) {
1032                 tmp[len] = '\0';
1033                 strlcpy(to, strim(tmp), available);
1034         }
1035
1036         return len;
1037 }
1038
1039 static ssize_t dtf_read_channel(struct file *file, char __user *buf,
1040                 size_t count, loff_t *ppos)
1041 {
1042         struct dmatest_info *info = file->private_data;
1043         return simple_read_from_buffer(buf, count, ppos,
1044                         info->dbgfs_params.channel,
1045                         strlen(info->dbgfs_params.channel));
1046 }
1047
1048 static ssize_t dtf_write_channel(struct file *file, const char __user *buf,
1049                 size_t size, loff_t *ppos)
1050 {
1051         struct dmatest_info *info = file->private_data;
1052         return dtf_write_string(info->dbgfs_params.channel,
1053                                 sizeof(info->dbgfs_params.channel),
1054                                 ppos, buf, size);
1055 }
1056
1057 static const struct file_operations dtf_channel_fops = {
1058         .read   = dtf_read_channel,
1059         .write  = dtf_write_channel,
1060         .open   = simple_open,
1061         .llseek = default_llseek,
1062 };
1063
1064 static ssize_t dtf_read_device(struct file *file, char __user *buf,
1065                 size_t count, loff_t *ppos)
1066 {
1067         struct dmatest_info *info = file->private_data;
1068         return simple_read_from_buffer(buf, count, ppos,
1069                         info->dbgfs_params.device,
1070                         strlen(info->dbgfs_params.device));
1071 }
1072
1073 static ssize_t dtf_write_device(struct file *file, const char __user *buf,
1074                 size_t size, loff_t *ppos)
1075 {
1076         struct dmatest_info *info = file->private_data;
1077         return dtf_write_string(info->dbgfs_params.device,
1078                                 sizeof(info->dbgfs_params.device),
1079                                 ppos, buf, size);
1080 }
1081
1082 static const struct file_operations dtf_device_fops = {
1083         .read   = dtf_read_device,
1084         .write  = dtf_write_device,
1085         .open   = simple_open,
1086         .llseek = default_llseek,
1087 };
1088
1089 static ssize_t dtf_read_run(struct file *file, char __user *user_buf,
1090                 size_t count, loff_t *ppos)
1091 {
1092         struct dmatest_info *info = file->private_data;
1093         char buf[3];
1094         struct dmatest_chan *dtc;
1095         bool alive = false;
1096
1097         mutex_lock(&info->lock);
1098         list_for_each_entry(dtc, &info->channels, node) {
1099                 struct dmatest_thread *thread;
1100
1101                 list_for_each_entry(thread, &dtc->threads, node) {
1102                         if (!thread->done) {
1103                                 alive = true;
1104                                 break;
1105                         }
1106                 }
1107         }
1108
1109         if (alive) {
1110                 buf[0] = 'Y';
1111         } else {
1112                 __stop_threaded_test(info);
1113                 buf[0] = 'N';
1114         }
1115
1116         mutex_unlock(&info->lock);
1117         buf[1] = '\n';
1118         buf[2] = 0x00;
1119         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1120 }
1121
1122 static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
1123                 size_t count, loff_t *ppos)
1124 {
1125         struct dmatest_info *info = file->private_data;
1126         char buf[16];
1127         bool bv;
1128         int ret = 0;
1129
1130         if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1))))
1131                 return -EFAULT;
1132
1133         if (strtobool(buf, &bv) == 0) {
1134                 mutex_lock(&info->lock);
1135                 ret = __restart_threaded_test(info, bv);
1136                 mutex_unlock(&info->lock);
1137         }
1138
1139         return ret ? ret : count;
1140 }
1141
1142 static const struct file_operations dtf_run_fops = {
1143         .read   = dtf_read_run,
1144         .write  = dtf_write_run,
1145         .open   = simple_open,
1146         .llseek = default_llseek,
1147 };
1148
1149 static int dtf_results_show(struct seq_file *sf, void *data)
1150 {
1151         struct dmatest_info *info = sf->private;
1152         struct dmatest_result *result;
1153         struct dmatest_thread_result *tr;
1154         unsigned int i;
1155
1156         mutex_lock(&info->results_lock);
1157         list_for_each_entry(result, &info->results, node) {
1158                 list_for_each_entry(tr, &result->results, node) {
1159                         seq_printf(sf, "%s\n",
1160                                 thread_result_get(result->name, tr));
1161                         if (tr->type == DMATEST_ET_VERIFY_BUF) {
1162                                 for (i = 0; i < tr->vr->error_count; i++) {
1163                                         seq_printf(sf, "\t%s\n",
1164                                                 verify_result_get_one(tr->vr, i));
1165                                 }
1166                         }
1167                 }
1168         }
1169
1170         mutex_unlock(&info->results_lock);
1171         return 0;
1172 }
1173
1174 static int dtf_results_open(struct inode *inode, struct file *file)
1175 {
1176         return single_open(file, dtf_results_show, inode->i_private);
1177 }
1178
1179 static const struct file_operations dtf_results_fops = {
1180         .open           = dtf_results_open,
1181         .read           = seq_read,
1182         .llseek         = seq_lseek,
1183         .release        = single_release,
1184 };
1185
1186 static int dmatest_register_dbgfs(struct dmatest_info *info)
1187 {
1188         struct dentry *d;
1189         struct dmatest_params *params = &info->dbgfs_params;
1190         int ret = -ENOMEM;
1191
1192         d = debugfs_create_dir("dmatest", NULL);
1193         if (IS_ERR(d))
1194                 return PTR_ERR(d);
1195         if (!d)
1196                 goto err_root;
1197
1198         info->root = d;
1199
1200         /* Copy initial values */
1201         memcpy(params, &info->params, sizeof(*params));
1202
1203         /* Test parameters */
1204
1205         d = debugfs_create_u32("test_buf_size", S_IWUSR | S_IRUGO, info->root,
1206                                (u32 *)&params->buf_size);
1207         if (IS_ERR_OR_NULL(d))
1208                 goto err_node;
1209
1210         d = debugfs_create_file("channel", S_IRUGO | S_IWUSR, info->root,
1211                                 info, &dtf_channel_fops);
1212         if (IS_ERR_OR_NULL(d))
1213                 goto err_node;
1214
1215         d = debugfs_create_file("device", S_IRUGO | S_IWUSR, info->root,
1216                                 info, &dtf_device_fops);
1217         if (IS_ERR_OR_NULL(d))
1218                 goto err_node;
1219
1220         d = debugfs_create_u32("threads_per_chan", S_IWUSR | S_IRUGO, info->root,
1221                                (u32 *)&params->threads_per_chan);
1222         if (IS_ERR_OR_NULL(d))
1223                 goto err_node;
1224
1225         d = debugfs_create_u32("max_channels", S_IWUSR | S_IRUGO, info->root,
1226                                (u32 *)&params->max_channels);
1227         if (IS_ERR_OR_NULL(d))
1228                 goto err_node;
1229
1230         d = debugfs_create_u32("iterations", S_IWUSR | S_IRUGO, info->root,
1231                                (u32 *)&params->iterations);
1232         if (IS_ERR_OR_NULL(d))
1233                 goto err_node;
1234
1235         d = debugfs_create_u32("xor_sources", S_IWUSR | S_IRUGO, info->root,
1236                                (u32 *)&params->xor_sources);
1237         if (IS_ERR_OR_NULL(d))
1238                 goto err_node;
1239
1240         d = debugfs_create_u32("pq_sources", S_IWUSR | S_IRUGO, info->root,
1241                                (u32 *)&params->pq_sources);
1242         if (IS_ERR_OR_NULL(d))
1243                 goto err_node;
1244
1245         d = debugfs_create_u32("timeout", S_IWUSR | S_IRUGO, info->root,
1246                                (u32 *)&params->timeout);
1247         if (IS_ERR_OR_NULL(d))
1248                 goto err_node;
1249
1250         /* Run or stop threaded test */
1251         d = debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root,
1252                                 info, &dtf_run_fops);
1253         if (IS_ERR_OR_NULL(d))
1254                 goto err_node;
1255
1256         /* Results of test in progress */
1257         d = debugfs_create_file("results", S_IRUGO, info->root, info,
1258                                 &dtf_results_fops);
1259         if (IS_ERR_OR_NULL(d))
1260                 goto err_node;
1261
1262         return 0;
1263
1264 err_node:
1265         debugfs_remove_recursive(info->root);
1266 err_root:
1267         pr_err("dmatest: Failed to initialize debugfs\n");
1268         return ret;
1269 }
1270
1271 static int __init dmatest_init(void)
1272 {
1273         struct dmatest_info *info = &test_info;
1274         struct dmatest_params *params = &info->params;
1275         int ret;
1276
1277         memset(info, 0, sizeof(*info));
1278
1279         mutex_init(&info->lock);
1280         INIT_LIST_HEAD(&info->channels);
1281
1282         mutex_init(&info->results_lock);
1283         INIT_LIST_HEAD(&info->results);
1284
1285         /* Set default parameters */
1286         params->buf_size = test_buf_size;
1287         strlcpy(params->channel, test_channel, sizeof(params->channel));
1288         strlcpy(params->device, test_device, sizeof(params->device));
1289         params->threads_per_chan = threads_per_chan;
1290         params->max_channels = max_channels;
1291         params->iterations = iterations;
1292         params->xor_sources = xor_sources;
1293         params->pq_sources = pq_sources;
1294         params->timeout = timeout;
1295
1296         ret = dmatest_register_dbgfs(info);
1297         if (ret)
1298                 return ret;
1299
1300 #ifdef MODULE
1301         return 0;
1302 #else
1303         return run_threaded_test(info);
1304 #endif
1305 }
1306 /* when compiled-in wait for drivers to load first */
1307 late_initcall(dmatest_init);
1308
1309 static void __exit dmatest_exit(void)
1310 {
1311         struct dmatest_info *info = &test_info;
1312
1313         debugfs_remove_recursive(info->root);
1314         stop_threaded_test(info);
1315         result_free(info, NULL);
1316 }
1317 module_exit(dmatest_exit);
1318
1319 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1320 MODULE_LICENSE("GPL v2");