drivers/misc/genwqe/card_dev.c

   1 /**
   2  * IBM Accelerator Family 'GenWQE'
   3  *
   4  * (C) Copyright IBM Corp. 2013
   5  *
   6  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
   7  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
   8  * Author: Michael Jung <mijung@de.ibm.com>
   9  * Author: Michael Ruettger <michael@ibmra.de>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License (version 2 only)
  13  * as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU General Public License for more details.
  19  */
  20
  21 /*
  22  * Character device representation of the GenWQE device. This allows
  23  * user-space applications to communicate with the card.
  24  */
  25
  26 #include <linux/kernel.h>
  27 #include <linux/types.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/string.h>
  31 #include <linux/fs.h>
  32 #include <linux/sched.h>
  33 #include <linux/wait.h>
  34 #include <linux/delay.h>
  35 #include <linux/atomic.h>
  36
  37 #include "card_base.h"
  38 #include "card_ddcb.h"
  39
  40 static int genwqe_open_files(struct genwqe_dev *cd)
  41 {
  42         int rc;
  43         unsigned long flags;
  44
  45         spin_lock_irqsave(&cd->file_lock, flags);
  46         rc = list_empty(&cd->file_list);
  47         spin_unlock_irqrestore(&cd->file_lock, flags);
  48         return !rc;
  49 }
  50
  51 static void genwqe_add_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
  52 {
  53         unsigned long flags;
  54
  55         cfile->owner = current;
  56         spin_lock_irqsave(&cd->file_lock, flags);
  57         list_add(&cfile->list, &cd->file_list);
  58         spin_unlock_irqrestore(&cd->file_lock, flags);
  59 }
  60
  61 static int genwqe_del_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
  62 {
  63         unsigned long flags;
  64
  65         spin_lock_irqsave(&cd->file_lock, flags);
  66         list_del(&cfile->list);
  67         spin_unlock_irqrestore(&cd->file_lock, flags);
  68
  69         return 0;
  70 }
  71
  72 static void genwqe_add_pin(struct genwqe_file *cfile, struct dma_mapping *m)
  73 {
  74         unsigned long flags;
  75
  76         spin_lock_irqsave(&cfile->pin_lock, flags);
  77         list_add(&m->pin_list, &cfile->pin_list);
  78         spin_unlock_irqrestore(&cfile->pin_lock, flags);
  79 }
  80
  81 static int genwqe_del_pin(struct genwqe_file *cfile, struct dma_mapping *m)
  82 {
  83         unsigned long flags;
  84
  85         spin_lock_irqsave(&cfile->pin_lock, flags);
  86         list_del(&m->pin_list);
  87         spin_unlock_irqrestore(&cfile->pin_lock, flags);
  88
  89         return 0;
  90 }
  91
  92 /**
  93  * genwqe_search_pin() - Search for the mapping for a userspace address
  94  * @cfile:      Descriptor of opened file
  95  * @u_addr:     User virtual address
  96  * @size:       Size of buffer
  97  * @dma_addr:   DMA address to be updated
  98  *
  99  * Return: Pointer to the corresponding mapping NULL if not found
 100  */
 101 static struct dma_mapping *genwqe_search_pin(struct genwqe_file *cfile,
 102                                             unsigned long u_addr,
 103                                             unsigned int size,
 104                                             void **virt_addr)
 105 {
 106         unsigned long flags;
 107         struct dma_mapping *m;
 108
 109         spin_lock_irqsave(&cfile->pin_lock, flags);
 110
 111         list_for_each_entry(m, &cfile->pin_list, pin_list) {
 112                 if ((((u64)m->u_vaddr) <= (u_addr)) &&
 113                     (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
 114
 115                         if (virt_addr)
 116                                 *virt_addr = m->k_vaddr +
 117                                         (u_addr - (u64)m->u_vaddr);
 118
 119                         spin_unlock_irqrestore(&cfile->pin_lock, flags);
 120                         return m;
 121                 }
 122         }
 123         spin_unlock_irqrestore(&cfile->pin_lock, flags);
 124         return NULL;
 125 }
 126
 127 static void __genwqe_add_mapping(struct genwqe_file *cfile,
 128                               struct dma_mapping *dma_map)
 129 {
 130         unsigned long flags;
 131
 132         spin_lock_irqsave(&cfile->map_lock, flags);
 133         list_add(&dma_map->card_list, &cfile->map_list);
 134         spin_unlock_irqrestore(&cfile->map_lock, flags);
 135 }
 136
 137 static void __genwqe_del_mapping(struct genwqe_file *cfile,
 138                               struct dma_mapping *dma_map)
 139 {
 140         unsigned long flags;
 141
 142         spin_lock_irqsave(&cfile->map_lock, flags);
 143         list_del(&dma_map->card_list);
 144         spin_unlock_irqrestore(&cfile->map_lock, flags);
 145 }
 146
 147
 148 /**
 149  * __genwqe_search_mapping() - Search for the mapping for a userspace address
 150  * @cfile:      descriptor of opened file
 151  * @u_addr:     user virtual address
 152  * @size:       size of buffer
 153  * @dma_addr:   DMA address to be updated
 154  * Return: Pointer to the corresponding mapping NULL if not found
 155  */
 156 static struct dma_mapping *__genwqe_search_mapping(struct genwqe_file *cfile,
 157                                                    unsigned long u_addr,
 158                                                    unsigned int size,
 159                                                    dma_addr_t *dma_addr,
 160                                                    void **virt_addr)
 161 {
 162         unsigned long flags;
 163         struct dma_mapping *m;
 164         struct pci_dev *pci_dev = cfile->cd->pci_dev;
 165
 166         spin_lock_irqsave(&cfile->map_lock, flags);
 167         list_for_each_entry(m, &cfile->map_list, card_list) {
 168
 169                 if ((((u64)m->u_vaddr) <= (u_addr)) &&
 170                     (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
 171
 172                         /* match found: current is as expected and
 173                            addr is in range */
 174                         if (dma_addr)
 175                                 *dma_addr = m->dma_addr +
 176                                         (u_addr - (u64)m->u_vaddr);
 177
 178                         if (virt_addr)
 179                                 *virt_addr = m->k_vaddr +
 180                                         (u_addr - (u64)m->u_vaddr);
 181
 182                         spin_unlock_irqrestore(&cfile->map_lock, flags);
 183                         return m;
 184                 }
 185         }
 186         spin_unlock_irqrestore(&cfile->map_lock, flags);
 187
 188         dev_err(&pci_dev->dev,
 189                 "[%s] Entry not found: u_addr=%lx, size=%x\n",
 190                 __func__, u_addr, size);
 191
 192         return NULL;
 193 }
 194
 195 static void genwqe_remove_mappings(struct genwqe_file *cfile)
 196 {
 197         int i = 0;
 198         struct list_head *node, *next;
 199         struct dma_mapping *dma_map;
 200         struct genwqe_dev *cd = cfile->cd;
 201         struct pci_dev *pci_dev = cfile->cd->pci_dev;
 202
 203         list_for_each_safe(node, next, &cfile->map_list) {
 204                 dma_map = list_entry(node, struct dma_mapping, card_list);
 205
 206                 list_del_init(&dma_map->card_list);
 207
 208                 /*
 209                  * This is really a bug, because those things should
 210                  * have been already tidied up.
 211                  *
 212                  * GENWQE_MAPPING_RAW should have been removed via mmunmap().
 213                  * GENWQE_MAPPING_SGL_TEMP should be removed by tidy up code.
 214                  */
 215                 dev_err(&pci_dev->dev,
 216                         "[%s] %d. cleanup mapping: u_vaddr=%p "
 217                         "u_kaddr=%016lx dma_addr=%llx\n", __func__, i++,
 218                         dma_map->u_vaddr, (unsigned long)dma_map->k_vaddr,
 219                         dma_map->dma_addr);
 220
 221                 if (dma_map->type == GENWQE_MAPPING_RAW) {
 222                         /* we allocated this dynamically */
 223                         __genwqe_free_consistent(cd, dma_map->size,
 224                                                 dma_map->k_vaddr,
 225                                                 dma_map->dma_addr);
 226                         kfree(dma_map);
 227                 } else if (dma_map->type == GENWQE_MAPPING_SGL_TEMP) {
 228                         /* we use dma_map statically from the request */
 229                         genwqe_user_vunmap(cd, dma_map, NULL);
 230                 }
 231         }
 232 }
 233
 234 static void genwqe_remove_pinnings(struct genwqe_file *cfile)
 235 {
 236         struct list_head *node, *next;
 237         struct dma_mapping *dma_map;
 238         struct genwqe_dev *cd = cfile->cd;
 239
 240         list_for_each_safe(node, next, &cfile->pin_list) {
 241                 dma_map = list_entry(node, struct dma_mapping, pin_list);
 242
 243                 /*
 244                  * This is not a bug, because a killed processed might
 245                  * not call the unpin ioctl, which is supposed to free
 246                  * the resources.
 247                  *
 248                  * Pinnings are dymically allocated and need to be
 249                  * deleted.
 250                  */
 251                 list_del_init(&dma_map->pin_list);
 252                 genwqe_user_vunmap(cd, dma_map, NULL);
 253                 kfree(dma_map);
 254         }
 255 }
 256
 257 /**
 258  * genwqe_kill_fasync() - Send signal to all processes with open GenWQE files
 259  *
 260  * E.g. genwqe_send_signal(cd, SIGIO);
 261  */
 262 static int genwqe_kill_fasync(struct genwqe_dev *cd, int sig)
 263 {
 264         unsigned int files = 0;
 265         unsigned long flags;
 266         struct genwqe_file *cfile;
 267
 268         spin_lock_irqsave(&cd->file_lock, flags);
 269         list_for_each_entry(cfile, &cd->file_list, list) {
 270                 if (cfile->async_queue)
 271                         kill_fasync(&cfile->async_queue, sig, POLL_HUP);
 272                 files++;
 273         }
 274         spin_unlock_irqrestore(&cd->file_lock, flags);
 275         return files;
 276 }
 277
 278 static int genwqe_force_sig(struct genwqe_dev *cd, int sig)
 279 {
 280         unsigned int files = 0;
 281         unsigned long flags;
 282         struct genwqe_file *cfile;
 283
 284         spin_lock_irqsave(&cd->file_lock, flags);
 285         list_for_each_entry(cfile, &cd->file_list, list) {
 286                 force_sig(sig, cfile->owner);
 287                 files++;
 288         }
 289         spin_unlock_irqrestore(&cd->file_lock, flags);
 290         return files;
 291 }
 292
 293 /**
 294  * genwqe_open() - file open
 295  * @inode:      file system information
 296  * @filp:       file handle
 297  *
 298  * This function is executed whenever an application calls
 299  * open("/dev/genwqe",..).
 300  *
 301  * Return: 0 if successful or <0 if errors
 302  */
 303 static int genwqe_open(struct inode *inode, struct file *filp)
 304 {
 305         struct genwqe_dev *cd;
 306         struct genwqe_file *cfile;
 307         struct pci_dev *pci_dev;
 308
 309         cfile = kzalloc(sizeof(*cfile), GFP_KERNEL);
 310         if (cfile == NULL)
 311                 return -ENOMEM;
 312
 313         cd = container_of(inode->i_cdev, struct genwqe_dev, cdev_genwqe);
 314         pci_dev = cd->pci_dev;
 315         cfile->cd = cd;
 316         cfile->filp = filp;
 317         cfile->client = NULL;
 318
 319         spin_lock_init(&cfile->map_lock);  /* list of raw memory allocations */
 320         INIT_LIST_HEAD(&cfile->map_list);
 321
 322         spin_lock_init(&cfile->pin_lock);  /* list of user pinned memory */
 323         INIT_LIST_HEAD(&cfile->pin_list);
 324
 325         filp->private_data = cfile;
 326
 327         genwqe_add_file(cd, cfile);
 328         return 0;
 329 }
 330
 331 /**
 332  * genwqe_fasync() - Setup process to receive SIGIO.
 333  * @fd:        file descriptor
 334  * @filp:      file handle
 335  * @mode:      file mode
 336  *
 337  * Sending a signal is working as following:
 338  *
 339  * if (cdev->async_queue)
 340  *         kill_fasync(&cdev->async_queue, SIGIO, POLL_IN);
 341  *
 342  * Some devices also implement asynchronous notification to indicate
 343  * when the device can be written; in this case, of course,
 344  * kill_fasync must be called with a mode of POLL_OUT.
 345  */
 346 static int genwqe_fasync(int fd, struct file *filp, int mode)
 347 {
 348         struct genwqe_file *cdev = (struct genwqe_file *)filp->private_data;
 349         return fasync_helper(fd, filp, mode, &cdev->async_queue);
 350 }
 351
 352
 353 /**
 354  * genwqe_release() - file close
 355  * @inode:      file system information
 356  * @filp:       file handle
 357  *
 358  * This function is executed whenever an application calls 'close(fd_genwqe)'
 359  *
 360  * Return: always 0
 361  */
 362 static int genwqe_release(struct inode *inode, struct file *filp)
 363 {
 364         struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
 365         struct genwqe_dev *cd = cfile->cd;
 366
 367         /* there must be no entries in these lists! */
 368         genwqe_remove_mappings(cfile);
 369         genwqe_remove_pinnings(cfile);
 370
 371         /* remove this filp from the asynchronously notified filp's */
 372         genwqe_fasync(-1, filp, 0);
 373
 374         /*
 375          * For this to work we must not release cd when this cfile is
 376          * not yet released, otherwise the list entry is invalid,
 377          * because the list itself gets reinstantiated!
 378          */
 379         genwqe_del_file(cd, cfile);
 380         kfree(cfile);
 381         return 0;
 382 }
 383
 384 static void genwqe_vma_open(struct vm_area_struct *vma)
 385 {
 386         /* nothing ... */
 387 }
 388
 389 /**
 390  * genwqe_vma_close() - Called each time when vma is unmapped
 391  *
 392  * Free memory which got allocated by GenWQE mmap().
 393  */
 394 static void genwqe_vma_close(struct vm_area_struct *vma)
 395 {
 396         unsigned long vsize = vma->vm_end - vma->vm_start;
 397         struct inode *inode = vma->vm_file->f_dentry->d_inode;
 398         struct dma_mapping *dma_map;
 399         struct genwqe_dev *cd = container_of(inode->i_cdev, struct genwqe_dev,
 400                                             cdev_genwqe);
 401         struct pci_dev *pci_dev = cd->pci_dev;
 402         dma_addr_t d_addr = 0;
 403         struct genwqe_file *cfile = vma->vm_private_data;
 404
 405         dma_map = __genwqe_search_mapping(cfile, vma->vm_start, vsize,
 406                                          &d_addr, NULL);
 407         if (dma_map == NULL) {
 408                 dev_err(&pci_dev->dev,
 409                         "  [%s] err: mapping not found: v=%lx, p=%lx s=%lx\n",
 410                         __func__, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT,
 411                         vsize);
 412                 return;
 413         }
 414         __genwqe_del_mapping(cfile, dma_map);
 415         __genwqe_free_consistent(cd, dma_map->size, dma_map->k_vaddr,
 416                                  dma_map->dma_addr);
 417         kfree(dma_map);
 418 }
 419
 420 static struct vm_operations_struct genwqe_vma_ops = {
 421         .open   = genwqe_vma_open,
 422         .close  = genwqe_vma_close,
 423 };
 424
 425 /**
 426  * genwqe_mmap() - Provide contignous buffers to userspace
 427  *
 428  * We use mmap() to allocate contignous buffers used for DMA
 429  * transfers. After the buffer is allocated we remap it to user-space
 430  * and remember a reference to our dma_mapping data structure, where
 431  * we store the associated DMA address and allocated size.
 432  *
 433  * When we receive a DDCB execution request with the ATS bits set to
 434  * plain buffer, we lookup our dma_mapping list to find the
 435  * corresponding DMA address for the associated user-space address.
 436  */
 437 static int genwqe_mmap(struct file *filp, struct vm_area_struct *vma)
 438 {
 439         int rc;
 440         unsigned long pfn, vsize = vma->vm_end - vma->vm_start;
 441         struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
 442         struct genwqe_dev *cd = cfile->cd;
 443         struct dma_mapping *dma_map;
 444
 445         if (vsize == 0)
 446                 return -EINVAL;
 447
 448         if (get_order(vsize) > MAX_ORDER)
 449                 return -ENOMEM;
 450
 451         dma_map = kzalloc(sizeof(struct dma_mapping), GFP_ATOMIC);
 452         if (dma_map == NULL)
 453                 return -ENOMEM;
 454
 455         genwqe_mapping_init(dma_map, GENWQE_MAPPING_RAW);
 456         dma_map->u_vaddr = (void *)vma->vm_start;
 457         dma_map->size = vsize;
 458         dma_map->nr_pages = DIV_ROUND_UP(vsize, PAGE_SIZE);
 459         dma_map->k_vaddr = __genwqe_alloc_consistent(cd, vsize,
 460                                                      &dma_map->dma_addr);
 461         if (dma_map->k_vaddr == NULL) {
 462                 rc = -ENOMEM;
 463                 goto free_dma_map;
 464         }
 465
 466         if (capable(CAP_SYS_ADMIN) && (vsize > sizeof(dma_addr_t)))
 467                 *(dma_addr_t *)dma_map->k_vaddr = dma_map->dma_addr;
 468
 469         pfn = virt_to_phys(dma_map->k_vaddr) >> PAGE_SHIFT;
 470         rc = remap_pfn_range(vma,
 471                              vma->vm_start,
 472                              pfn,
 473                              vsize,
 474                              vma->vm_page_prot);
 475         if (rc != 0) {
 476                 rc = -EFAULT;
 477                 goto free_dma_mem;
 478         }
 479
 480         vma->vm_private_data = cfile;
 481         vma->vm_ops = &genwqe_vma_ops;
 482         __genwqe_add_mapping(cfile, dma_map);
 483
 484         return 0;
 485
 486  free_dma_mem:
 487         __genwqe_free_consistent(cd, dma_map->size,
 488                                 dma_map->k_vaddr,
 489                                 dma_map->dma_addr);
 490  free_dma_map:
 491         kfree(dma_map);
 492         return rc;
 493 }
 494
 495 /**
 496  * do_flash_update() - Excute flash update (write image or CVPD)
 497  * @cd:        genwqe device
 498  * @load:      details about image load
 499  *
 500  * Return: 0 if successful
 501  */
 502
 503 #define FLASH_BLOCK     0x40000 /* we use 256k blocks */
 504
 505 static int do_flash_update(struct genwqe_file *cfile,
 506                            struct genwqe_bitstream *load)
 507 {
 508         int rc = 0;
 509         int blocks_to_flash;
 510         u64 dma_addr, flash = 0;
 511         size_t tocopy = 0;
 512         u8 __user *buf;
 513         u8 *xbuf;
 514         u32 crc;
 515         u8 cmdopts;
 516         struct genwqe_dev *cd = cfile->cd;
 517         struct pci_dev *pci_dev = cd->pci_dev;
 518
 519         if ((load->size & 0x3) != 0) {
 520                 dev_err(&pci_dev->dev,
 521                         "err: buf %d bytes not 4 bytes aligned!\n",
 522                         load->size);
 523                 return -EINVAL;
 524         }
 525         if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0) {
 526                 dev_err(&pci_dev->dev,
 527                         "err: buf is not page aligned!\n");
 528                 return -EINVAL;
 529         }
 530
 531         /* FIXME Bits have changed for new service layer! */
 532         switch ((char)load->partition) {
 533         case '0':
 534                 cmdopts = 0x14;
 535                 break;          /* download/erase_first/part_0 */
 536         case '1':
 537                 cmdopts = 0x1C;
 538                 break;          /* download/erase_first/part_1 */
 539         case 'v':               /* cmdopts = 0x0c (VPD) */
 540         default:
 541                 dev_err(&pci_dev->dev,
 542                         "err: invalid partition %02x!\n", load->partition);
 543                 return -EINVAL;
 544         }
 545         dev_info(&pci_dev->dev,
 546                  "[%s] start flash update UID: 0x%x size: %u bytes part: %c\n",
 547                  __func__, load->uid, load->size, (char)load->partition);
 548
 549         buf = (u8 __user *)load->data_addr;
 550         xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
 551         if (xbuf == NULL) {
 552                 dev_err(&pci_dev->dev, "err: no memory\n");
 553                 return -ENOMEM;
 554         }
 555
 556         blocks_to_flash = load->size / FLASH_BLOCK;
 557         while (load->size) {
 558                 struct genwqe_ddcb_cmd *req;
 559
 560                 /*
 561                  * We must be 4 byte aligned. Buffer must be 0 appened
 562                  * to have defined values when calculating CRC.
 563                  */
 564                 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
 565
 566                 rc = copy_from_user(xbuf, buf, tocopy);
 567                 if (rc) {
 568                         dev_err(&pci_dev->dev,
 569                                 "err: could not copy all data rc=%d\n", rc);
 570                         goto free_buffer;
 571                 }
 572                 crc = genwqe_crc32(xbuf, tocopy, 0xffffffff);
 573
 574                 dev_info(&pci_dev->dev,
 575                          "[%s] DMA: 0x%llx CRC: %08x SZ: %ld %d\n",
 576                         __func__, dma_addr, crc, tocopy, blocks_to_flash);
 577
 578                 /* prepare DDCB for SLU process */
 579                 req = ddcb_requ_alloc();
 580                 if (req == NULL) {
 581                         rc = -ENOMEM;
 582                         goto free_buffer;
 583                 }
 584
 585                 req->cmd = SLCMD_MOVE_FLASH;
 586                 req->cmdopts = cmdopts;
 587
 588                 /* prepare invariant values */
 589                 if (genwqe_get_slu_id(cd) <= 0x2) {
 590                         *(u64 *)&req->__asiv[0]  = cpu_to_be64(dma_addr);
 591                         *(u64 *)&req->__asiv[8]  = cpu_to_be64(tocopy);
 592                         *(u64 *)&req->__asiv[16] = cpu_to_be64(flash);
 593                         *(u32 *)&req->__asiv[24] = cpu_to_be32(0);
 594                         req->__asiv[24]        = load->uid;
 595                         *(u32 *)&req->__asiv[28] = cpu_to_be32(crc);
 596
 597                         /* for simulation only */
 598                         *(u64 *)&req->__asiv[88] = cpu_to_be64(load->slu_id);
 599                         *(u64 *)&req->__asiv[96] = cpu_to_be64(load->app_id);
 600                         req->asiv_length = 32; /* bytes included in crc calc */
 601                 } else {        /* setup DDCB for ATS architecture */
 602                         *(u64 *)&req->asiv[0]  = cpu_to_be64(dma_addr);
 603                         *(u32 *)&req->asiv[8]  = cpu_to_be32(tocopy);
 604                         *(u32 *)&req->asiv[12] = cpu_to_be32(0); /* resvd */
 605                         *(u64 *)&req->asiv[16] = cpu_to_be64(flash);
 606                         *(u32 *)&req->asiv[24] = cpu_to_be32(load->uid<<24);
 607                         *(u32 *)&req->asiv[28] = cpu_to_be32(crc);
 608
 609                         /* for simulation only */
 610                         *(u64 *)&req->asiv[80] = cpu_to_be64(load->slu_id);
 611                         *(u64 *)&req->asiv[88] = cpu_to_be64(load->app_id);
 612
 613                         req->ats = cpu_to_be64(0x4ULL << 44);   /* Rd only */
 614                         req->asiv_length = 40; /* bytes included in crc calc */
 615                 }
 616                 req->asv_length  = 8;
 617
 618                 /* For Genwqe5 we get back the calculated CRC */
 619                 *(u64 *)&req->asv[0] = 0ULL;                    /* 0x80 */
 620
 621                 rc = __genwqe_execute_raw_ddcb(cd, req);
 622
 623                 load->retc = req->retc;
 624                 load->attn = req->attn;
 625                 load->progress = req->progress;
 626
 627                 if (rc < 0) {
 628                         dev_err(&pci_dev->dev,
 629                                 "  [%s] DDCB returned (RETC=%x ATTN=%x "
 630                                 "PROG=%x rc=%d)\n", __func__, req->retc,
 631                                 req->attn, req->progress, rc);
 632
 633                         ddcb_requ_free(req);
 634                         goto free_buffer;
 635                 }
 636
 637                 if (req->retc != DDCB_RETC_COMPLETE) {
 638                         dev_info(&pci_dev->dev,
 639                                  "  [%s] DDCB returned (RETC=%x ATTN=%x "
 640                                  "PROG=%x)\n", __func__, req->retc,
 641                                  req->attn, req->progress);
 642
 643                         rc = -EIO;
 644                         ddcb_requ_free(req);
 645                         goto free_buffer;
 646                 }
 647
 648                 load->size  -= tocopy;
 649                 flash += tocopy;
 650                 buf += tocopy;
 651                 blocks_to_flash--;
 652                 ddcb_requ_free(req);
 653         }
 654
 655  free_buffer:
 656         __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
 657         return rc;
 658 }
 659
 660 static int do_flash_read(struct genwqe_file *cfile,
 661                          struct genwqe_bitstream *load)
 662 {
 663         int rc, blocks_to_flash;
 664         u64 dma_addr, flash = 0;
 665         size_t tocopy = 0;
 666         u8 __user *buf;
 667         u8 *xbuf;
 668         u8 cmdopts;
 669         struct genwqe_dev *cd = cfile->cd;
 670         struct pci_dev *pci_dev = cd->pci_dev;
 671         struct genwqe_ddcb_cmd *cmd;
 672
 673         if ((load->size & 0x3) != 0) {
 674                 dev_err(&pci_dev->dev,
 675                         "err: buf size %d bytes not 4 bytes aligned!\n",
 676                         load->size);
 677                 return -EINVAL;
 678         }
 679         if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0) {
 680                 dev_err(&pci_dev->dev, "err: buf is not page aligned!\n");
 681                 return -EINVAL;
 682         }
 683
 684         /* FIXME Bits have changed for new service layer! */
 685         switch ((char)load->partition) {
 686         case '0':
 687                 cmdopts = 0x12;
 688                 break;          /* upload/part_0 */
 689         case '1':
 690                 cmdopts = 0x1A;
 691                 break;          /* upload/part_1 */
 692         case 'v':
 693         default:
 694                 dev_err(&pci_dev->dev,
 695                         "err: invalid partition %02x!\n", load->partition);
 696                 return -EINVAL;
 697         }
 698         dev_info(&pci_dev->dev,
 699                  "[%s] start flash read UID: 0x%x size: %u bytes part: %c\n",
 700                  __func__, load->uid, load->size, (char)load->partition);
 701
 702         buf = (u8 __user *)load->data_addr;
 703         xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
 704         if (xbuf == NULL) {
 705                 dev_err(&pci_dev->dev, "err: no memory\n");
 706                 return -ENOMEM;
 707         }
 708
 709         blocks_to_flash = load->size / FLASH_BLOCK;
 710         while (load->size) {
 711                 /*
 712                  * We must be 4 byte aligned. Buffer must be 0 appened
 713                  * to have defined values when calculating CRC.
 714                  */
 715                 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
 716
 717                 dev_info(&pci_dev->dev,
 718                          "[%s] DMA: 0x%llx SZ: %ld %d\n",
 719                          __func__, dma_addr, tocopy, blocks_to_flash);
 720
 721                 /* prepare DDCB for SLU process */
 722                 cmd = ddcb_requ_alloc();
 723                 if (cmd == NULL) {
 724                         rc = -ENOMEM;
 725                         goto free_buffer;
 726                 }
 727                 cmd->cmd = SLCMD_MOVE_FLASH;
 728                 cmd->cmdopts = cmdopts;
 729
 730                 /* prepare invariant values */
 731                 if (genwqe_get_slu_id(cd) <= 0x2) {
 732                         *(u64 *)&cmd->__asiv[0]  = cpu_to_be64(dma_addr);
 733                         *(u64 *)&cmd->__asiv[8]  = cpu_to_be64(tocopy);
 734                         *(u64 *)&cmd->__asiv[16] = cpu_to_be64(flash);
 735                         *(u32 *)&cmd->__asiv[24] = cpu_to_be32(0);
 736                         cmd->__asiv[24] = load->uid;
 737                         *(u32 *)&cmd->__asiv[28] = cpu_to_be32(0)  /* CRC */;
 738                         cmd->asiv_length = 32; /* bytes included in crc calc */
 739                 } else {        /* setup DDCB for ATS architecture */
 740                         *(u64 *)&cmd->asiv[0]  = cpu_to_be64(dma_addr);
 741                         *(u32 *)&cmd->asiv[8]  = cpu_to_be32(tocopy);
 742                         *(u32 *)&cmd->asiv[12] = cpu_to_be32(0); /* resvd */
 743                         *(u64 *)&cmd->asiv[16] = cpu_to_be64(flash);
 744                         *(u32 *)&cmd->asiv[24] = cpu_to_be32(load->uid<<24);
 745                         *(u32 *)&cmd->asiv[28] = cpu_to_be32(0); /* CRC */
 746                         cmd->ats = cpu_to_be64(0x5ULL << 44);   /* rd/wr */
 747                         cmd->asiv_length = 40; /* bytes included in crc calc */
 748                 }
 749                 cmd->asv_length  = 8;
 750
 751                 /* we only get back the calculated CRC */
 752                 *(u64 *)&cmd->asv[0] = 0ULL;    /* 0x80 */
 753
 754                 rc = __genwqe_execute_raw_ddcb(cd, cmd);
 755
 756                 load->retc = cmd->retc;
 757                 load->attn = cmd->attn;
 758                 load->progress = cmd->progress;
 759
 760                 if ((rc < 0) && (rc != -EBADMSG)) {
 761                         dev_err(&pci_dev->dev,
 762                                 "  [%s] DDCB returned (RETC=%x ATTN=%x "
 763                                 "PROG=%x rc=%d)\n", __func__, cmd->retc,
 764                                 cmd->attn, cmd->progress, rc);
 765                         ddcb_requ_free(cmd);
 766                         goto free_buffer;
 767                 }
 768
 769                 rc = copy_to_user(buf, xbuf, tocopy);
 770                 if (rc) {
 771                         dev_err(&pci_dev->dev,
 772                                 "  [%s] copy data to user failed rc=%d\n",
 773                                 __func__, rc);
 774                         rc = -EIO;
 775                         ddcb_requ_free(cmd);
 776                         goto free_buffer;
 777                 }
 778
 779                 /* We know that we can get retc 0x104 with CRC err */
 780                 if (((cmd->retc == DDCB_RETC_FAULT) &&
 781                      (cmd->attn != 0x02)) ||  /* Normally ignore CRC error */
 782                     ((cmd->retc == DDCB_RETC_COMPLETE) &&
 783                      (cmd->attn != 0x00))) {  /* Everything was fine */
 784                         dev_err(&pci_dev->dev,
 785                                 "  [%s] DDCB returned (RETC=%x ATTN=%x "
 786                                 "PROG=%x rc=%d)\n", __func__, cmd->retc,
 787                                 cmd->attn, cmd->progress, rc);
 788                         rc = -EIO;
 789                         ddcb_requ_free(cmd);
 790                         goto free_buffer;
 791                 }
 792
 793                 load->size  -= tocopy;
 794                 flash += tocopy;
 795                 buf += tocopy;
 796                 blocks_to_flash--;
 797                 ddcb_requ_free(cmd);
 798         }
 799         rc = 0;
 800
 801  free_buffer:
 802         __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
 803         return rc;
 804 }
 805
 806 static int genwqe_pin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
 807 {
 808         int rc;
 809         struct genwqe_dev *cd = cfile->cd;
 810         struct pci_dev *pci_dev = cfile->cd->pci_dev;
 811         struct dma_mapping *dma_map;
 812         unsigned long map_addr;
 813         unsigned long map_size;
 814
 815         if ((m->addr == 0x0) || (m->size == 0))
 816                 return -EINVAL;
 817
 818         map_addr = (m->addr & PAGE_MASK);
 819         map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
 820
 821         dma_map = kzalloc(sizeof(struct dma_mapping), GFP_ATOMIC);
 822         if (dma_map == NULL)
 823                 return -ENOMEM;
 824
 825         genwqe_mapping_init(dma_map, GENWQE_MAPPING_SGL_PINNED);
 826         rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size, NULL);
 827         if (rc != 0) {
 828                 dev_err(&pci_dev->dev,
 829                         "[%s] genwqe_user_vmap rc=%d\n", __func__, rc);
 830                 return rc;
 831         }
 832
 833         genwqe_add_pin(cfile, dma_map);
 834         return 0;
 835 }
 836
 837 static int genwqe_unpin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
 838 {
 839         struct genwqe_dev *cd = cfile->cd;
 840         struct dma_mapping *dma_map;
 841         unsigned long map_addr;
 842         unsigned long map_size;
 843
 844         if (m->addr == 0x0)
 845                 return -EINVAL;
 846
 847         map_addr = (m->addr & PAGE_MASK);
 848         map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
 849
 850         dma_map = genwqe_search_pin(cfile, map_addr, map_size, NULL);
 851         if (dma_map == NULL)
 852                 return -ENOENT;
 853
 854         genwqe_del_pin(cfile, dma_map);
 855         genwqe_user_vunmap(cd, dma_map, NULL);
 856         kfree(dma_map);
 857         return 0;
 858 }
 859
 860 /**
 861  * ddcb_cmd_cleanup() - Remove dynamically created fixup entries
 862  *
 863  * Only if there are any. Pinnings are not removed.
 864  */
 865 static int ddcb_cmd_cleanup(struct genwqe_file *cfile, struct ddcb_requ *req)
 866 {
 867         unsigned int i;
 868         struct dma_mapping *dma_map;
 869         struct genwqe_dev *cd = cfile->cd;
 870
 871         for (i = 0; i < DDCB_FIXUPS; i++) {
 872                 dma_map = &req->dma_mappings[i];
 873
 874                 if (dma_mapping_used(dma_map)) {
 875                         __genwqe_del_mapping(cfile, dma_map);
 876                         genwqe_user_vunmap(cd, dma_map, req);
 877                 }
 878                 if (req->sgl[i] != NULL) {
 879                         genwqe_free_sgl(cd, req->sgl[i],
 880                                        req->sgl_dma_addr[i],
 881                                        req->sgl_size[i]);
 882                         req->sgl[i] = NULL;
 883                         req->sgl_dma_addr[i] = 0x0;
 884                         req->sgl_size[i] = 0;
 885                 }
 886
 887         }
 888         return 0;
 889 }
 890
 891 /**
 892  * ddcb_cmd_fixups() - Establish DMA fixups/sglists for user memory references
 893  *
 894  * Before the DDCB gets executed we need to handle the fixups. We
 895  * replace the user-space addresses with DMA addresses or do
 896  * additional setup work e.g. generating a scatter-gather list which
 897  * is used to describe the memory referred to in the fixup.
 898  */
 899 static int ddcb_cmd_fixups(struct genwqe_file *cfile, struct ddcb_requ *req)
 900 {
 901         int rc;
 902         unsigned int asiv_offs, i;
 903         struct genwqe_dev *cd = cfile->cd;
 904         struct genwqe_ddcb_cmd *cmd = &req->cmd;
 905         struct dma_mapping *m;
 906         struct pci_dev *pci_dev = cd->pci_dev;
 907         const char *type = "UNKNOWN";
 908
 909         for (i = 0, asiv_offs = 0x00; asiv_offs <= 0x58;
 910              i++, asiv_offs += 0x08) {
 911
 912                 u64 u_addr, d_addr;
 913                 u32 u_size = 0;
 914                 unsigned long ats_flags;
 915
 916                 ats_flags = ATS_GET_FLAGS(be64_to_cpu(cmd->ats), asiv_offs);
 917
 918                 switch (ats_flags) {
 919
 920                 case ATS_TYPE_DATA:
 921                         break;  /* nothing to do here */
 922
 923                 case ATS_TYPE_FLAT_RDWR:
 924                 case ATS_TYPE_FLAT_RD: {
 925                         u_addr = be64_to_cpu(*((u64 *)&cmd->
 926                                                asiv[asiv_offs]));
 927                         u_size = be32_to_cpu(*((u32 *)&cmd->
 928                                                asiv[asiv_offs + 0x08]));
 929
 930                         /*
 931                          * No data available. Ignore u_addr in this
 932                          * case and set addr to 0. Hardware must not
 933                          * fetch the buffer.
 934                          */
 935                         if (u_size == 0x0) {
 936                                 *((u64 *)&cmd->asiv[asiv_offs]) =
 937                                         cpu_to_be64(0x0);
 938                                 break;
 939                         }
 940
 941                         m = __genwqe_search_mapping(cfile, u_addr, u_size,
 942                                                    &d_addr, NULL);
 943                         if (m == NULL) {
 944                                 rc = -EFAULT;
 945                                 goto err_out;
 946                         }
 947
 948                         *((u64 *)&cmd->asiv[asiv_offs]) = cpu_to_be64(d_addr);
 949                         break;
 950                 }
 951
 952                 case ATS_TYPE_SGL_RDWR:
 953                 case ATS_TYPE_SGL_RD: {
 954                         int page_offs, nr_pages, offs;
 955
 956                         u_addr = be64_to_cpu(*((u64 *)&cmd->asiv[asiv_offs]));
 957                         u_size = be32_to_cpu(*((u32 *)&cmd->asiv[asiv_offs +
 958                                                                  0x08]));
 959
 960                         /*
 961                          * No data available. Ignore u_addr in this
 962                          * case and set addr to 0. Hardware must not
 963                          * fetch the empty sgl.
 964                          */
 965                         if (u_size == 0x0) {
 966                                 *((u64 *)&cmd->asiv[asiv_offs]) =
 967                                         cpu_to_be64(0x0);
 968                                 break;
 969                         }
 970
 971                         m = genwqe_search_pin(cfile, u_addr, u_size, NULL);
 972                         if (m != NULL) {
 973                                 type = "PINNING";
 974                                 page_offs = (u_addr -
 975                                              (u64)m->u_vaddr)/PAGE_SIZE;
 976                         } else {
 977                                 type = "MAPPING";
 978                                 m = &req->dma_mappings[i];
 979
 980                                 genwqe_mapping_init(m,
 981                                                     GENWQE_MAPPING_SGL_TEMP);
 982                                 rc = genwqe_user_vmap(cd, m, (void *)u_addr,
 983                                                       u_size, req);
 984                                 if (rc != 0)
 985                                         goto err_out;
 986
 987                                 __genwqe_add_mapping(cfile, m);
 988                                 page_offs = 0;
 989                         }
 990
 991                         offs = offset_in_page(u_addr);
 992                         nr_pages = DIV_ROUND_UP(offs + u_size, PAGE_SIZE);
 993
 994                         /* create genwqe style scatter gather list */
 995                         req->sgl[i] = genwqe_alloc_sgl(cd, m->nr_pages,
 996                                                       &req->sgl_dma_addr[i],
 997                                                       &req->sgl_size[i]);
 998                         if (req->sgl[i] == NULL) {
 999                                 rc = -ENOMEM;
1000                                 goto err_out;
1001                         }
1002                         genwqe_setup_sgl(cd, offs, u_size,
1003                                         req->sgl[i],
1004                                         req->sgl_dma_addr[i],
1005                                         req->sgl_size[i],
1006                                         m->dma_list,
1007                                         page_offs,
1008                                         nr_pages);
1009
1010                         *((u64 *)&cmd->asiv[asiv_offs]) =
1011                                 cpu_to_be64(req->sgl_dma_addr[i]);
1012
1013                         break;
1014                 }
1015                 default:
1016                         dev_err(&pci_dev->dev,
1017                                 "[%s] err: invalid ATS flags %01lx\n",
1018                                 __func__, ats_flags);
1019                         rc = -EINVAL;
1020                         goto err_out;
1021                 }
1022         }
1023         return 0;
1024
1025  err_out:
1026         dev_err(&pci_dev->dev, "[%s] err: rc=%d\n", __func__, rc);
1027         ddcb_cmd_cleanup(cfile, req);
1028         return rc;
1029 }
1030
1031 /**
1032  * genwqe_execute_ddcb() - Execute DDCB using userspace address fixups
1033  *
1034  * The code will build up the translation tables or lookup the
1035  * contignous memory allocation table to find the right translations
1036  * and DMA addresses.
1037  */
1038 static int genwqe_execute_ddcb(struct genwqe_file *cfile,
1039                                struct genwqe_ddcb_cmd *cmd)
1040 {
1041         int rc;
1042         struct genwqe_dev *cd = cfile->cd;
1043         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
1044
1045         rc = ddcb_cmd_fixups(cfile, req);
1046         if (rc != 0)
1047                 return rc;
1048
1049         rc = __genwqe_execute_raw_ddcb(cd, cmd);
1050         ddcb_cmd_cleanup(cfile, req);
1051         return rc;
1052 }
1053
1054 static int do_execute_ddcb(struct genwqe_file *cfile,
1055                            unsigned long arg, int raw)
1056 {
1057         int rc;
1058         struct genwqe_ddcb_cmd *cmd;
1059         struct ddcb_requ *req;
1060         struct genwqe_dev *cd = cfile->cd;
1061         struct pci_dev *pci_dev = cd->pci_dev;
1062
1063         cmd = ddcb_requ_alloc();
1064         if (cmd == NULL)
1065                 return -ENOMEM;
1066
1067         req = container_of(cmd, struct ddcb_requ, cmd);
1068
1069         if (copy_from_user(cmd, (void __user *)arg, sizeof(*cmd))) {
1070                 dev_err(&pci_dev->dev,
1071                         "err: could not copy params from user\n");
1072                 ddcb_requ_free(cmd);
1073                 return -EFAULT;
1074         }
1075
1076         if (!raw)
1077                 rc = genwqe_execute_ddcb(cfile, cmd);
1078         else
1079                 rc = __genwqe_execute_raw_ddcb(cd, cmd);
1080
1081         /* Copy back only the modifed fields. Do not copy ASIV
1082            back since the copy got modified by the driver. */
1083         if (copy_to_user((void __user *)arg, cmd,
1084                          sizeof(*cmd) - DDCB_ASIV_LENGTH)) {
1085                 dev_err(&pci_dev->dev,
1086                         "err: could not copy params to user\n");
1087                 ddcb_requ_free(cmd);
1088                 return -EFAULT;
1089         }
1090
1091         ddcb_requ_free(cmd);
1092         return rc;
1093 }
1094
1095 /**
1096  * genwqe_ioctl() - IO control
1097  * @filp:       file handle
1098  * @cmd:        command identifier (passed from user)
1099  * @arg:        argument (passed from user)
1100  *
1101  * Return: 0 success
1102  */
1103 static long genwqe_ioctl(struct file *filp, unsigned int cmd,
1104                          unsigned long arg)
1105 {
1106         int rc = 0;
1107         struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
1108         struct genwqe_dev *cd = cfile->cd;
1109         struct genwqe_reg_io __user *io;
1110         u64 val;
1111         u32 reg_offs;
1112         struct pci_dev *pci_dev = cd->pci_dev;
1113
1114         if (_IOC_TYPE(cmd) != GENWQE_IOC_CODE) {
1115                 dev_err(&pci_dev->dev, "err: ioctl code does not match!\n");
1116                 return -EINVAL;
1117         }
1118
1119         switch (cmd) {
1120
1121         case GENWQE_GET_CARD_STATE:
1122                 put_user(cd->card_state, (enum genwqe_card_state __user *)arg);
1123                 return 0;
1124
1125                 /* Register access */
1126         case GENWQE_READ_REG64: {
1127                 io = (struct genwqe_reg_io __user *)arg;
1128
1129                 if (get_user(reg_offs, &io->num)) {
1130                         dev_err(&pci_dev->dev, "err: reg read64\n");
1131                         return -EFAULT;
1132                 }
1133                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1134                         return -EINVAL;
1135
1136                 val = __genwqe_readq(cd, reg_offs);
1137                 put_user(val, &io->val64);
1138                 return 0;
1139         }
1140
1141         case GENWQE_WRITE_REG64: {
1142                 io = (struct genwqe_reg_io __user *)arg;
1143
1144                 if (!capable(CAP_SYS_ADMIN))
1145                         return -EPERM;
1146
1147                 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1148                         return -EPERM;
1149
1150                 if (get_user(reg_offs, &io->num)) {
1151                         dev_err(&pci_dev->dev, "err: reg write64\n");
1152                         return -EFAULT;
1153                 }
1154                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1155                         return -EINVAL;
1156
1157                 if (get_user(val, &io->val64)) {
1158                         dev_err(&pci_dev->dev, "err: reg write64\n");
1159                         return -EFAULT;
1160                 }
1161                 __genwqe_writeq(cd, reg_offs, val);
1162                 return 0;
1163         }
1164
1165         case GENWQE_READ_REG32: {
1166                 io = (struct genwqe_reg_io __user *)arg;
1167
1168                 if (get_user(reg_offs, &io->num)) {
1169                         dev_err(&pci_dev->dev, "err: reg read32\n");
1170                         return -EFAULT;
1171                 }
1172                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1173                         return -EINVAL;
1174
1175                 val = __genwqe_readl(cd, reg_offs);
1176                 put_user(val, &io->val64);
1177                 return 0;
1178         }
1179
1180         case GENWQE_WRITE_REG32: {
1181                 io = (struct genwqe_reg_io __user *)arg;
1182
1183                 if (!capable(CAP_SYS_ADMIN))
1184                         return -EPERM;
1185
1186                 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1187                         return -EPERM;
1188
1189                 if (get_user(reg_offs, &io->num)) {
1190                         dev_err(&pci_dev->dev, "err: reg write32\n");
1191                         return -EFAULT;
1192                 }
1193                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1194                         return -EINVAL;
1195
1196                 if (get_user(val, &io->val64)) {
1197                         dev_err(&pci_dev->dev, "err: reg write32\n");
1198                         return -EFAULT;
1199                 }
1200                 __genwqe_writel(cd, reg_offs, val);
1201                 return 0;
1202         }
1203
1204                 /* Flash update/reading */
1205         case GENWQE_SLU_UPDATE: {
1206                 struct genwqe_bitstream load;
1207
1208                 if (!genwqe_is_privileged(cd))
1209                         return -EPERM;
1210
1211                 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1212                         return -EPERM;
1213
1214                 if (copy_from_user(&load, (void __user *)arg, sizeof(load))) {
1215                         dev_err(&pci_dev->dev,
1216                                 "err: could not copy params from user\n");
1217                         return -EFAULT;
1218                 }
1219                 rc = do_flash_update(cfile, &load);
1220
1221                 if (copy_to_user((void __user *)arg, &load, sizeof(load))) {
1222                         dev_err(&pci_dev->dev,
1223                                 "err: could not copy params to user\n");
1224                         return -EFAULT;
1225                 }
1226                 dev_info(&pci_dev->dev, "[%s] rc=%d\n", __func__, rc);
1227                 return rc;
1228         }
1229
1230         case GENWQE_SLU_READ: {
1231                 struct genwqe_bitstream load;
1232
1233                 if (!genwqe_is_privileged(cd))
1234                         return -EPERM;
1235
1236                 if (genwqe_flash_readback_fails(cd))
1237                         return -ENOSPC;  /* known to fail for old versions */
1238
1239                 if (copy_from_user(&load, (void __user *)arg, sizeof(load))) {
1240                         dev_err(&pci_dev->dev,
1241                                 "err: could not copy params from user\n");
1242                         return -EFAULT;
1243                 }
1244                 rc = do_flash_read(cfile, &load);
1245
1246                 if (copy_to_user((void __user *)arg, &load, sizeof(load))) {
1247                         dev_err(&pci_dev->dev,
1248                                 "err: could not copy params to user\n");
1249                         return -EFAULT;
1250                 }
1251                 dev_info(&pci_dev->dev, "[%s] rc=%d\n", __func__, rc);
1252                 return rc;
1253         }
1254
1255                 /* memory pinning and unpinning */
1256         case GENWQE_PIN_MEM: {
1257                 struct genwqe_mem m;
1258
1259                 if (copy_from_user(&m, (void __user *)arg, sizeof(m))) {
1260                         dev_err(&pci_dev->dev,
1261                                 "err: could not copy params from user\n");
1262                         return -EFAULT;
1263                 }
1264                 return genwqe_pin_mem(cfile, &m);
1265         }
1266
1267         case GENWQE_UNPIN_MEM: {
1268                 struct genwqe_mem m;
1269
1270                 if (copy_from_user(&m, (void __user *)arg, sizeof(m))) {
1271                         dev_err(&pci_dev->dev,
1272                                 "err: could not copy params from user\n");
1273                         return -EFAULT;
1274                 }
1275                 return genwqe_unpin_mem(cfile, &m);
1276         }
1277
1278                 /* launch an DDCB and wait for completion */
1279         case GENWQE_EXECUTE_DDCB:
1280                 return do_execute_ddcb(cfile, arg, 0);
1281
1282         case GENWQE_EXECUTE_RAW_DDCB: {
1283
1284                 if (!capable(CAP_SYS_ADMIN)) {
1285                         dev_err(&pci_dev->dev,
1286                                 "err: must be superuser execute raw DDCB!\n");
1287                         return -EPERM;
1288                 }
1289                 return do_execute_ddcb(cfile, arg, 1);
1290         }
1291
1292         default:
1293                 pr_err("unknown ioctl %x/%lx**\n", cmd, arg);
1294                 return -EINVAL;
1295         }
1296
1297         return rc;
1298 }
1299
1300 #if defined(CONFIG_COMPAT)
1301 /**
1302  * genwqe_compat_ioctl() - Compatibility ioctl
1303  *
1304  * Called whenever a 32-bit process running under a 64-bit kernel
1305  * performs an ioctl on /dev/genwqe<n>_card.
1306  *
1307  * @filp:        file pointer.
1308  * @cmd:         command.
1309  * @arg:         user argument.
1310  * Return:       zero on success or negative number on failure.
1311  */
1312 static long genwqe_compat_ioctl(struct file *filp, unsigned int cmd,
1313                                 unsigned long arg)
1314 {
1315         return genwqe_ioctl(filp, cmd, arg);
1316 }
1317 #endif /* defined(CONFIG_COMPAT) */
1318
1319 static const struct file_operations genwqe_fops = {
1320         .owner          = THIS_MODULE,
1321         .open           = genwqe_open,
1322         .fasync         = genwqe_fasync,
1323         .mmap           = genwqe_mmap,
1324         .unlocked_ioctl = genwqe_ioctl,
1325 #if defined(CONFIG_COMPAT)
1326         .compat_ioctl   = genwqe_compat_ioctl,
1327 #endif
1328         .release        = genwqe_release,
1329 };
1330
1331 static int genwqe_device_initialized(struct genwqe_dev *cd)
1332 {
1333         return cd->dev != NULL;
1334 }
1335
1336 /**
1337  * genwqe_device_create() - Create and configure genwqe char device
1338  * @cd:      genwqe device descriptor
1339  *
1340  * This function must be called before we create any more genwqe
1341  * character devices, because it is allocating the major and minor
1342  * number which are supposed to be used by the client drivers.
1343  */
1344 int genwqe_device_create(struct genwqe_dev *cd)
1345 {
1346         int rc;
1347         struct pci_dev *pci_dev = cd->pci_dev;
1348
1349         /*
1350          * Here starts the individual setup per client. It must
1351          * initialize its own cdev data structure with its own fops.
1352          * The appropriate devnum needs to be created. The ranges must
1353          * not overlap.
1354          */
1355         rc = alloc_chrdev_region(&cd->devnum_genwqe, 0,
1356                                  GENWQE_MAX_MINOR, GENWQE_DEVNAME);
1357         if (rc < 0) {
1358                 dev_err(&pci_dev->dev, "err: alloc_chrdev_region failed\n");
1359                 goto err_dev;
1360         }
1361
1362         cdev_init(&cd->cdev_genwqe, &genwqe_fops);
1363         cd->cdev_genwqe.owner = THIS_MODULE;
1364
1365         rc = cdev_add(&cd->cdev_genwqe, cd->devnum_genwqe, 1);
1366         if (rc < 0) {
1367                 dev_err(&pci_dev->dev, "err: cdev_add failed\n");
1368                 goto err_add;
1369         }
1370
1371         /*
1372          * Finally the device in /dev/... must be created. The rule is
1373          * to use card%d_clientname for each created device.
1374          */
1375         cd->dev = device_create_with_groups(cd->class_genwqe,
1376                                             &cd->pci_dev->dev,
1377                                             cd->devnum_genwqe, cd,
1378                                             genwqe_attribute_groups,
1379                                             GENWQE_DEVNAME "%u_card",
1380                                             cd->card_idx);
1381         if (cd->dev == NULL) {
1382                 rc = -ENODEV;
1383                 goto err_cdev;
1384         }
1385
1386         rc = genwqe_init_debugfs(cd);
1387         if (rc != 0)
1388                 goto err_debugfs;
1389
1390         return 0;
1391
1392  err_debugfs:
1393         device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1394  err_cdev:
1395         cdev_del(&cd->cdev_genwqe);
1396  err_add:
1397         unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1398  err_dev:
1399         cd->dev = NULL;
1400         return rc;
1401 }
1402
1403 static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)
1404 {
1405         int rc;
1406         unsigned int i;
1407         struct pci_dev *pci_dev = cd->pci_dev;
1408
1409         if (!genwqe_open_files(cd))
1410                 return 0;
1411
1412         dev_warn(&pci_dev->dev, "[%s] send SIGIO and wait ...\n", __func__);
1413
1414         rc = genwqe_kill_fasync(cd, SIGIO);
1415         if (rc > 0) {
1416                 /* give kill_timeout seconds to close file descriptors ... */
1417                 for (i = 0; (i < genwqe_kill_timeout) &&
1418                              genwqe_open_files(cd); i++) {
1419                         dev_info(&pci_dev->dev, "  %d sec ...", i);
1420
1421                         cond_resched();
1422                         msleep(1000);
1423                 }
1424
1425                 /* if no open files we can safely continue, else ... */
1426                 if (!genwqe_open_files(cd))
1427                         return 0;
1428
1429                 dev_warn(&pci_dev->dev,
1430                          "[%s] send SIGKILL and wait ...\n", __func__);
1431
1432                 rc = genwqe_force_sig(cd, SIGKILL); /* force terminate */
1433                 if (rc) {
1434                         /* Give kill_timout more seconds to end processes */
1435                         for (i = 0; (i < genwqe_kill_timeout) &&
1436                                      genwqe_open_files(cd); i++) {
1437                                 dev_warn(&pci_dev->dev, "  %d sec ...", i);
1438
1439                                 cond_resched();
1440                                 msleep(1000);
1441                         }
1442                 }
1443         }
1444         return 0;
1445 }
1446
1447 /**
1448  * genwqe_device_remove() - Remove genwqe's char device
1449  *
1450  * This function must be called after the client devices are removed
1451  * because it will free the major/minor number range for the genwqe
1452  * drivers.
1453  *
1454  * This function must be robust enough to be called twice.
1455  */
1456 int genwqe_device_remove(struct genwqe_dev *cd)
1457 {
1458         int rc;
1459         struct pci_dev *pci_dev = cd->pci_dev;
1460
1461         if (!genwqe_device_initialized(cd))
1462                 return 1;
1463
1464         genwqe_inform_and_stop_processes(cd);
1465
1466         /*
1467          * We currently do wait until all filedescriptors are
1468          * closed. This leads to a problem when we abort the
1469          * application which will decrease this reference from
1470          * 1/unused to 0/illegal and not from 2/used 1/empty.
1471          */
1472         rc = atomic_read(&cd->cdev_genwqe.kobj.kref.refcount);
1473         if (rc != 1) {
1474                 dev_err(&pci_dev->dev,
1475                         "[%s] err: cdev_genwqe...refcount=%d\n", __func__, rc);
1476                 panic("Fatal err: cannot free resources with pending references!");
1477         }
1478
1479         genqwe_exit_debugfs(cd);
1480         device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1481         cdev_del(&cd->cdev_genwqe);
1482         unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1483         cd->dev = NULL;
1484
1485         return 0;
1486 }