drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/slab.h>
  25 #include <linux/list.h>
  26 #include <linux/types.h>
  27 #include <linux/printk.h>
  28 #include <linux/bitops.h>
  29 #include <linux/sched.h>
  30 #include "kfd_priv.h"
  31 #include "kfd_device_queue_manager.h"
  32 #include "kfd_mqd_manager.h"
  33 #include "cik_regs.h"
  34 #include "kfd_kernel_queue.h"
  35
  36 /* Size of the per-pipe EOP queue */
  37 #define CIK_HPD_EOP_BYTES_LOG2 11
  38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
  39
  40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
  41                                         unsigned int pasid, unsigned int vmid);
  42
  43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
  44                                         struct queue *q,
  45                                         struct qcm_process_device *qpd);
  46
  47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  49
  50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
  51                                         struct queue *q,
  52                                         struct qcm_process_device *qpd);
  53
  54 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
  55                                 unsigned int sdma_queue_id);
  56
  57 static inline
  58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
  59 {
  60         if (type == KFD_QUEUE_TYPE_SDMA)
  61                 return KFD_MQD_TYPE_SDMA;
  62         return KFD_MQD_TYPE_CP;
  63 }
  64
  65 unsigned int get_first_pipe(struct device_queue_manager *dqm)
  66 {
  67         BUG_ON(!dqm || !dqm->dev);
  68         return dqm->dev->shared_resources.first_compute_pipe;
  69 }
  70
  71 unsigned int get_pipes_num(struct device_queue_manager *dqm)
  72 {
  73         BUG_ON(!dqm || !dqm->dev);
  74         return dqm->dev->shared_resources.compute_pipe_count;
  75 }
  76
  77 static inline unsigned int get_pipes_num_cpsch(void)
  78 {
  79         return PIPE_PER_ME_CP_SCHEDULING;
  80 }
  81
  82 void program_sh_mem_settings(struct device_queue_manager *dqm,
  83                                         struct qcm_process_device *qpd)
  84 {
  85         return dqm->dev->kfd2kgd->program_sh_mem_settings(
  86                                                 dqm->dev->kgd, qpd->vmid,
  87                                                 qpd->sh_mem_config,
  88                                                 qpd->sh_mem_ape1_base,
  89                                                 qpd->sh_mem_ape1_limit,
  90                                                 qpd->sh_mem_bases);
  91 }
  92
  93 static int allocate_vmid(struct device_queue_manager *dqm,
  94                         struct qcm_process_device *qpd,
  95                         struct queue *q)
  96 {
  97         int bit, allocated_vmid;
  98
  99         if (dqm->vmid_bitmap == 0)
 100                 return -ENOMEM;
 101
 102         bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
 103         clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 104
 105         /* Kaveri kfd vmid's starts from vmid 8 */
 106         allocated_vmid = bit + KFD_VMID_START_OFFSET;
 107         pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
 108         qpd->vmid = allocated_vmid;
 109         q->properties.vmid = allocated_vmid;
 110
 111         set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
 112         program_sh_mem_settings(dqm, qpd);
 113
 114         return 0;
 115 }
 116
 117 static void deallocate_vmid(struct device_queue_manager *dqm,
 118                                 struct qcm_process_device *qpd,
 119                                 struct queue *q)
 120 {
 121         int bit = qpd->vmid - KFD_VMID_START_OFFSET;
 122
 123         /* Release the vmid mapping */
 124         set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
 125
 126         set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 127         qpd->vmid = 0;
 128         q->properties.vmid = 0;
 129 }
 130
 131 static int create_queue_nocpsch(struct device_queue_manager *dqm,
 132                                 struct queue *q,
 133                                 struct qcm_process_device *qpd,
 134                                 int *allocated_vmid)
 135 {
 136         int retval;
 137
 138         BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
 139
 140         pr_debug("kfd: In func %s\n", __func__);
 141         print_queue(q);
 142
 143         mutex_lock(&dqm->lock);
 144
 145         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 146                 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
 147                                 dqm->total_queue_count);
 148                 mutex_unlock(&dqm->lock);
 149                 return -EPERM;
 150         }
 151
 152         if (list_empty(&qpd->queues_list)) {
 153                 retval = allocate_vmid(dqm, qpd, q);
 154                 if (retval != 0) {
 155                         mutex_unlock(&dqm->lock);
 156                         return retval;
 157                 }
 158         }
 159         *allocated_vmid = qpd->vmid;
 160         q->properties.vmid = qpd->vmid;
 161
 162         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 163                 retval = create_compute_queue_nocpsch(dqm, q, qpd);
 164         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 165                 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
 166
 167         if (retval != 0) {
 168                 if (list_empty(&qpd->queues_list)) {
 169                         deallocate_vmid(dqm, qpd, q);
 170                         *allocated_vmid = 0;
 171                 }
 172                 mutex_unlock(&dqm->lock);
 173                 return retval;
 174         }
 175
 176         list_add(&q->list, &qpd->queues_list);
 177         if (q->properties.is_active)
 178                 dqm->queue_count++;
 179
 180         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 181                 dqm->sdma_queue_count++;
 182
 183         /*
 184          * Unconditionally increment this counter, regardless of the queue's
 185          * type or whether the queue is active.
 186          */
 187         dqm->total_queue_count++;
 188         pr_debug("Total of %d queues are accountable so far\n",
 189                         dqm->total_queue_count);
 190
 191         mutex_unlock(&dqm->lock);
 192         return 0;
 193 }
 194
 195 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 196 {
 197         bool set;
 198         int pipe, bit, i;
 199
 200         set = false;
 201
 202         for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm);
 203                         pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) {
 204                 if (dqm->allocated_queues[pipe] != 0) {
 205                         bit = find_first_bit(
 206                                 (unsigned long *)&dqm->allocated_queues[pipe],
 207                                 QUEUES_PER_PIPE);
 208
 209                         clear_bit(bit,
 210                                 (unsigned long *)&dqm->allocated_queues[pipe]);
 211                         q->pipe = pipe;
 212                         q->queue = bit;
 213                         set = true;
 214                         break;
 215                 }
 216         }
 217
 218         if (set == false)
 219                 return -EBUSY;
 220
 221         pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
 222                                 __func__, q->pipe, q->queue);
 223         /* horizontal hqd allocation */
 224         dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
 225
 226         return 0;
 227 }
 228
 229 static inline void deallocate_hqd(struct device_queue_manager *dqm,
 230                                 struct queue *q)
 231 {
 232         set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
 233 }
 234
 235 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 236                                         struct queue *q,
 237                                         struct qcm_process_device *qpd)
 238 {
 239         int retval;
 240         struct mqd_manager *mqd;
 241
 242         BUG_ON(!dqm || !q || !qpd);
 243
 244         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 245         if (mqd == NULL)
 246                 return -ENOMEM;
 247
 248         retval = allocate_hqd(dqm, q);
 249         if (retval != 0)
 250                 return retval;
 251
 252         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 253                                 &q->gart_mqd_addr, &q->properties);
 254         if (retval != 0) {
 255                 deallocate_hqd(dqm, q);
 256                 return retval;
 257         }
 258
 259         pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
 260                         q->pipe,
 261                         q->queue);
 262
 263         retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
 264                         q->queue, (uint32_t __user *) q->properties.write_ptr);
 265         if (retval != 0) {
 266                 deallocate_hqd(dqm, q);
 267                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 268                 return retval;
 269         }
 270
 271         return 0;
 272 }
 273
 274 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 275                                 struct qcm_process_device *qpd,
 276                                 struct queue *q)
 277 {
 278         int retval;
 279         struct mqd_manager *mqd;
 280
 281         BUG_ON(!dqm || !q || !q->mqd || !qpd);
 282
 283         retval = 0;
 284
 285         pr_debug("kfd: In Func %s\n", __func__);
 286
 287         mutex_lock(&dqm->lock);
 288
 289         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 290                 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 291                 if (mqd == NULL) {
 292                         retval = -ENOMEM;
 293                         goto out;
 294                 }
 295                 deallocate_hqd(dqm, q);
 296         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 297                 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
 298                 if (mqd == NULL) {
 299                         retval = -ENOMEM;
 300                         goto out;
 301                 }
 302                 dqm->sdma_queue_count--;
 303                 deallocate_sdma_queue(dqm, q->sdma_id);
 304         } else {
 305                 pr_debug("q->properties.type is invalid (%d)\n",
 306                                 q->properties.type);
 307                 retval = -EINVAL;
 308                 goto out;
 309         }
 310
 311         retval = mqd->destroy_mqd(mqd, q->mqd,
 312                                 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 313                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
 314                                 q->pipe, q->queue);
 315
 316         if (retval != 0)
 317                 goto out;
 318
 319         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 320
 321         list_del(&q->list);
 322         if (list_empty(&qpd->queues_list))
 323                 deallocate_vmid(dqm, qpd, q);
 324         if (q->properties.is_active)
 325                 dqm->queue_count--;
 326
 327         /*
 328          * Unconditionally decrement this counter, regardless of the queue's
 329          * type
 330          */
 331         dqm->total_queue_count--;
 332         pr_debug("Total of %d queues are accountable so far\n",
 333                         dqm->total_queue_count);
 334
 335 out:
 336         mutex_unlock(&dqm->lock);
 337         return retval;
 338 }
 339
 340 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 341 {
 342         int retval;
 343         struct mqd_manager *mqd;
 344         bool prev_active = false;
 345
 346         BUG_ON(!dqm || !q || !q->mqd);
 347
 348         mutex_lock(&dqm->lock);
 349         mqd = dqm->ops.get_mqd_manager(dqm,
 350                         get_mqd_type_from_queue_type(q->properties.type));
 351         if (mqd == NULL) {
 352                 mutex_unlock(&dqm->lock);
 353                 return -ENOMEM;
 354         }
 355
 356         if (q->properties.is_active == true)
 357                 prev_active = true;
 358
 359         /*
 360          *
 361          * check active state vs. the previous state
 362          * and modify counter accordingly
 363          */
 364         retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
 365         if ((q->properties.is_active == true) && (prev_active == false))
 366                 dqm->queue_count++;
 367         else if ((q->properties.is_active == false) && (prev_active == true))
 368                 dqm->queue_count--;
 369
 370         if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
 371                 retval = execute_queues_cpsch(dqm, false);
 372
 373         mutex_unlock(&dqm->lock);
 374         return retval;
 375 }
 376
 377 static struct mqd_manager *get_mqd_manager_nocpsch(
 378                 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
 379 {
 380         struct mqd_manager *mqd;
 381
 382         BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
 383
 384         pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
 385
 386         mqd = dqm->mqds[type];
 387         if (!mqd) {
 388                 mqd = mqd_manager_init(type, dqm->dev);
 389                 if (mqd == NULL)
 390                         pr_err("kfd: mqd manager is NULL");
 391                 dqm->mqds[type] = mqd;
 392         }
 393
 394         return mqd;
 395 }
 396
 397 static int register_process_nocpsch(struct device_queue_manager *dqm,
 398                                         struct qcm_process_device *qpd)
 399 {
 400         struct device_process_node *n;
 401         int retval;
 402
 403         BUG_ON(!dqm || !qpd);
 404
 405         pr_debug("kfd: In func %s\n", __func__);
 406
 407         n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
 408         if (!n)
 409                 return -ENOMEM;
 410
 411         n->qpd = qpd;
 412
 413         mutex_lock(&dqm->lock);
 414         list_add(&n->list, &dqm->queues);
 415
 416         retval = dqm->ops_asic_specific.register_process(dqm, qpd);
 417
 418         dqm->processes_count++;
 419
 420         mutex_unlock(&dqm->lock);
 421
 422         return retval;
 423 }
 424
 425 static int unregister_process_nocpsch(struct device_queue_manager *dqm,
 426                                         struct qcm_process_device *qpd)
 427 {
 428         int retval;
 429         struct device_process_node *cur, *next;
 430
 431         BUG_ON(!dqm || !qpd);
 432
 433         BUG_ON(!list_empty(&qpd->queues_list));
 434
 435         pr_debug("kfd: In func %s\n", __func__);
 436
 437         retval = 0;
 438         mutex_lock(&dqm->lock);
 439
 440         list_for_each_entry_safe(cur, next, &dqm->queues, list) {
 441                 if (qpd == cur->qpd) {
 442                         list_del(&cur->list);
 443                         kfree(cur);
 444                         dqm->processes_count--;
 445                         goto out;
 446                 }
 447         }
 448         /* qpd not found in dqm list */
 449         retval = 1;
 450 out:
 451         mutex_unlock(&dqm->lock);
 452         return retval;
 453 }
 454
 455 static int
 456 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
 457                         unsigned int vmid)
 458 {
 459         uint32_t pasid_mapping;
 460
 461         pasid_mapping = (pasid == 0) ? 0 :
 462                 (uint32_t)pasid |
 463                 ATC_VMID_PASID_MAPPING_VALID;
 464
 465         return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
 466                                                 dqm->dev->kgd, pasid_mapping,
 467                                                 vmid);
 468 }
 469
 470 int init_pipelines(struct device_queue_manager *dqm,
 471                         unsigned int pipes_num, unsigned int first_pipe)
 472 {
 473         void *hpdptr;
 474         struct mqd_manager *mqd;
 475         unsigned int i, err, inx;
 476         uint64_t pipe_hpd_addr;
 477
 478         BUG_ON(!dqm || !dqm->dev);
 479
 480         pr_debug("kfd: In func %s\n", __func__);
 481
 482         /*
 483          * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
 484          * The driver never accesses this memory after zeroing it.
 485          * It doesn't even have to be saved/restored on suspend/resume
 486          * because it contains no data when there are no active queues.
 487          */
 488
 489         err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num,
 490                                         &dqm->pipeline_mem);
 491
 492         if (err) {
 493                 pr_err("kfd: error allocate vidmem num pipes: %d\n",
 494                         pipes_num);
 495                 return -ENOMEM;
 496         }
 497
 498         hpdptr = dqm->pipeline_mem->cpu_ptr;
 499         dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
 500
 501         memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
 502
 503         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 504         if (mqd == NULL) {
 505                 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 506                 return -ENOMEM;
 507         }
 508
 509         for (i = 0; i < pipes_num; i++) {
 510                 inx = i + first_pipe;
 511                 /*
 512                  * HPD buffer on GTT is allocated by amdkfd, no need to waste
 513                  * space in GTT for pipelines we don't initialize
 514                  */
 515                 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
 516                 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
 517                 /* = log2(bytes/4)-1 */
 518                 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx,
 519                                 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
 520         }
 521
 522         return 0;
 523 }
 524
 525 static int init_scheduler(struct device_queue_manager *dqm)
 526 {
 527         int retval;
 528
 529         BUG_ON(!dqm);
 530
 531         pr_debug("kfd: In %s\n", __func__);
 532
 533         retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm));
 534         return retval;
 535 }
 536
 537 static int initialize_nocpsch(struct device_queue_manager *dqm)
 538 {
 539         int i;
 540
 541         BUG_ON(!dqm);
 542
 543         pr_debug("kfd: In func %s num of pipes: %d\n",
 544                         __func__, get_pipes_num(dqm));
 545
 546         mutex_init(&dqm->lock);
 547         INIT_LIST_HEAD(&dqm->queues);
 548         dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 549         dqm->sdma_queue_count = 0;
 550         dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
 551                                         sizeof(unsigned int), GFP_KERNEL);
 552         if (!dqm->allocated_queues) {
 553                 mutex_destroy(&dqm->lock);
 554                 return -ENOMEM;
 555         }
 556
 557         for (i = 0; i < get_pipes_num(dqm); i++)
 558                 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
 559
 560         dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
 561         dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
 562
 563         init_scheduler(dqm);
 564         return 0;
 565 }
 566
 567 static void uninitialize_nocpsch(struct device_queue_manager *dqm)
 568 {
 569         int i;
 570
 571         BUG_ON(!dqm);
 572
 573         BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
 574
 575         kfree(dqm->allocated_queues);
 576         for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
 577                 kfree(dqm->mqds[i]);
 578         mutex_destroy(&dqm->lock);
 579         kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 580 }
 581
 582 static int start_nocpsch(struct device_queue_manager *dqm)
 583 {
 584         return 0;
 585 }
 586
 587 static int stop_nocpsch(struct device_queue_manager *dqm)
 588 {
 589         return 0;
 590 }
 591
 592 static int allocate_sdma_queue(struct device_queue_manager *dqm,
 593                                 unsigned int *sdma_queue_id)
 594 {
 595         int bit;
 596
 597         if (dqm->sdma_bitmap == 0)
 598                 return -ENOMEM;
 599
 600         bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
 601                                 CIK_SDMA_QUEUES);
 602
 603         clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
 604         *sdma_queue_id = bit;
 605
 606         return 0;
 607 }
 608
 609 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 610                                 unsigned int sdma_queue_id)
 611 {
 612         if (sdma_queue_id >= CIK_SDMA_QUEUES)
 613                 return;
 614         set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
 615 }
 616
 617 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 618                                         struct queue *q,
 619                                         struct qcm_process_device *qpd)
 620 {
 621         struct mqd_manager *mqd;
 622         int retval;
 623
 624         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
 625         if (!mqd)
 626                 return -ENOMEM;
 627
 628         retval = allocate_sdma_queue(dqm, &q->sdma_id);
 629         if (retval != 0)
 630                 return retval;
 631
 632         q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 633         q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
 634
 635         pr_debug("kfd: sdma id is:    %d\n", q->sdma_id);
 636         pr_debug("     sdma queue id: %d\n", q->properties.sdma_queue_id);
 637         pr_debug("     sdma engine id: %d\n", q->properties.sdma_engine_id);
 638
 639         dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
 640         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 641                                 &q->gart_mqd_addr, &q->properties);
 642         if (retval != 0) {
 643                 deallocate_sdma_queue(dqm, q->sdma_id);
 644                 return retval;
 645         }
 646
 647         retval = mqd->load_mqd(mqd, q->mqd, 0,
 648                                 0, NULL);
 649         if (retval != 0) {
 650                 deallocate_sdma_queue(dqm, q->sdma_id);
 651                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 652                 return retval;
 653         }
 654
 655         return 0;
 656 }
 657
 658 /*
 659  * Device Queue Manager implementation for cp scheduler
 660  */
 661
 662 static int set_sched_resources(struct device_queue_manager *dqm)
 663 {
 664         struct scheduling_resources res;
 665         unsigned int queue_num, queue_mask;
 666
 667         BUG_ON(!dqm);
 668
 669         pr_debug("kfd: In func %s\n", __func__);
 670
 671         queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
 672         queue_mask = (1 << queue_num) - 1;
 673         res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
 674         res.vmid_mask <<= KFD_VMID_START_OFFSET;
 675         res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
 676         res.gws_mask = res.oac_mask = res.gds_heap_base =
 677                                                 res.gds_heap_size = 0;
 678
 679         pr_debug("kfd: scheduling resources:\n"
 680                         "      vmid mask: 0x%8X\n"
 681                         "      queue mask: 0x%8llX\n",
 682                         res.vmid_mask, res.queue_mask);
 683
 684         return pm_send_set_resources(&dqm->packets, &res);
 685 }
 686
 687 static int initialize_cpsch(struct device_queue_manager *dqm)
 688 {
 689         int retval;
 690
 691         BUG_ON(!dqm);
 692
 693         pr_debug("kfd: In func %s num of pipes: %d\n",
 694                         __func__, get_pipes_num_cpsch());
 695
 696         mutex_init(&dqm->lock);
 697         INIT_LIST_HEAD(&dqm->queues);
 698         dqm->queue_count = dqm->processes_count = 0;
 699         dqm->sdma_queue_count = 0;
 700         dqm->active_runlist = false;
 701         retval = dqm->ops_asic_specific.initialize(dqm);
 702         if (retval != 0)
 703                 goto fail_init_pipelines;
 704
 705         return 0;
 706
 707 fail_init_pipelines:
 708         mutex_destroy(&dqm->lock);
 709         return retval;
 710 }
 711
 712 static int start_cpsch(struct device_queue_manager *dqm)
 713 {
 714         struct device_process_node *node;
 715         int retval;
 716
 717         BUG_ON(!dqm);
 718
 719         retval = 0;
 720
 721         retval = pm_init(&dqm->packets, dqm);
 722         if (retval != 0)
 723                 goto fail_packet_manager_init;
 724
 725         retval = set_sched_resources(dqm);
 726         if (retval != 0)
 727                 goto fail_set_sched_resources;
 728
 729         pr_debug("kfd: allocating fence memory\n");
 730
 731         /* allocate fence memory on the gart */
 732         retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
 733                                         &dqm->fence_mem);
 734
 735         if (retval != 0)
 736                 goto fail_allocate_vidmem;
 737
 738         dqm->fence_addr = dqm->fence_mem->cpu_ptr;
 739         dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
 740         list_for_each_entry(node, &dqm->queues, list)
 741                 if (node->qpd->pqm->process && dqm->dev)
 742                         kfd_bind_process_to_device(dqm->dev,
 743                                                 node->qpd->pqm->process);
 744
 745         execute_queues_cpsch(dqm, true);
 746
 747         return 0;
 748 fail_allocate_vidmem:
 749 fail_set_sched_resources:
 750         pm_uninit(&dqm->packets);
 751 fail_packet_manager_init:
 752         return retval;
 753 }
 754
 755 static int stop_cpsch(struct device_queue_manager *dqm)
 756 {
 757         struct device_process_node *node;
 758         struct kfd_process_device *pdd;
 759
 760         BUG_ON(!dqm);
 761
 762         destroy_queues_cpsch(dqm, true);
 763
 764         list_for_each_entry(node, &dqm->queues, list) {
 765                 pdd = qpd_to_pdd(node->qpd);
 766                 pdd->bound = false;
 767         }
 768         kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
 769         pm_uninit(&dqm->packets);
 770
 771         return 0;
 772 }
 773
 774 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 775                                         struct kernel_queue *kq,
 776                                         struct qcm_process_device *qpd)
 777 {
 778         BUG_ON(!dqm || !kq || !qpd);
 779
 780         pr_debug("kfd: In func %s\n", __func__);
 781
 782         mutex_lock(&dqm->lock);
 783         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 784                 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
 785                                 dqm->total_queue_count);
 786                 mutex_unlock(&dqm->lock);
 787                 return -EPERM;
 788         }
 789
 790         /*
 791          * Unconditionally increment this counter, regardless of the queue's
 792          * type or whether the queue is active.
 793          */
 794         dqm->total_queue_count++;
 795         pr_debug("Total of %d queues are accountable so far\n",
 796                         dqm->total_queue_count);
 797
 798         list_add(&kq->list, &qpd->priv_queue_list);
 799         dqm->queue_count++;
 800         qpd->is_debug = true;
 801         execute_queues_cpsch(dqm, false);
 802         mutex_unlock(&dqm->lock);
 803
 804         return 0;
 805 }
 806
 807 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 808                                         struct kernel_queue *kq,
 809                                         struct qcm_process_device *qpd)
 810 {
 811         BUG_ON(!dqm || !kq);
 812
 813         pr_debug("kfd: In %s\n", __func__);
 814
 815         mutex_lock(&dqm->lock);
 816         destroy_queues_cpsch(dqm, false);
 817         list_del(&kq->list);
 818         dqm->queue_count--;
 819         qpd->is_debug = false;
 820         execute_queues_cpsch(dqm, false);
 821         /*
 822          * Unconditionally decrement this counter, regardless of the queue's
 823          * type.
 824          */
 825         dqm->total_queue_count--;
 826         pr_debug("Total of %d queues are accountable so far\n",
 827                         dqm->total_queue_count);
 828         mutex_unlock(&dqm->lock);
 829 }
 830
 831 static void select_sdma_engine_id(struct queue *q)
 832 {
 833         static int sdma_id;
 834
 835         q->sdma_id = sdma_id;
 836         sdma_id = (sdma_id + 1) % 2;
 837 }
 838
 839 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 840                         struct qcm_process_device *qpd, int *allocate_vmid)
 841 {
 842         int retval;
 843         struct mqd_manager *mqd;
 844
 845         BUG_ON(!dqm || !q || !qpd);
 846
 847         retval = 0;
 848
 849         if (allocate_vmid)
 850                 *allocate_vmid = 0;
 851
 852         mutex_lock(&dqm->lock);
 853
 854         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 855                 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
 856                                 dqm->total_queue_count);
 857                 retval = -EPERM;
 858                 goto out;
 859         }
 860
 861         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 862                 select_sdma_engine_id(q);
 863
 864         mqd = dqm->ops.get_mqd_manager(dqm,
 865                         get_mqd_type_from_queue_type(q->properties.type));
 866
 867         if (mqd == NULL) {
 868                 mutex_unlock(&dqm->lock);
 869                 return -ENOMEM;
 870         }
 871
 872         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 873                                 &q->gart_mqd_addr, &q->properties);
 874         if (retval != 0)
 875                 goto out;
 876
 877         list_add(&q->list, &qpd->queues_list);
 878         if (q->properties.is_active) {
 879                 dqm->queue_count++;
 880                 retval = execute_queues_cpsch(dqm, false);
 881         }
 882
 883         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 884                         dqm->sdma_queue_count++;
 885         /*
 886          * Unconditionally increment this counter, regardless of the queue's
 887          * type or whether the queue is active.
 888          */
 889         dqm->total_queue_count++;
 890
 891         pr_debug("Total of %d queues are accountable so far\n",
 892                         dqm->total_queue_count);
 893
 894 out:
 895         mutex_unlock(&dqm->lock);
 896         return retval;
 897 }
 898
 899 static int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 900                                 unsigned int fence_value,
 901                                 unsigned long timeout)
 902 {
 903         BUG_ON(!fence_addr);
 904         timeout += jiffies;
 905
 906         while (*fence_addr != fence_value) {
 907                 if (time_after(jiffies, timeout)) {
 908                         pr_err("kfd: qcm fence wait loop timeout expired\n");
 909                         return -ETIME;
 910                 }
 911                 schedule();
 912         }
 913
 914         return 0;
 915 }
 916
 917 static int destroy_sdma_queues(struct device_queue_manager *dqm,
 918                                 unsigned int sdma_engine)
 919 {
 920         return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
 921                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false,
 922                         sdma_engine);
 923 }
 924
 925 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 926 {
 927         int retval;
 928
 929         BUG_ON(!dqm);
 930
 931         retval = 0;
 932
 933         if (lock)
 934                 mutex_lock(&dqm->lock);
 935         if (dqm->active_runlist == false)
 936                 goto out;
 937
 938         pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
 939                 dqm->sdma_queue_count);
 940
 941         if (dqm->sdma_queue_count > 0) {
 942                 destroy_sdma_queues(dqm, 0);
 943                 destroy_sdma_queues(dqm, 1);
 944         }
 945
 946         retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
 947                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
 948         if (retval != 0)
 949                 goto out;
 950
 951         *dqm->fence_addr = KFD_FENCE_INIT;
 952         pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
 953                                 KFD_FENCE_COMPLETED);
 954         /* should be timed out */
 955         amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
 956                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
 957         pm_release_ib(&dqm->packets);
 958         dqm->active_runlist = false;
 959
 960 out:
 961         if (lock)
 962                 mutex_unlock(&dqm->lock);
 963         return retval;
 964 }
 965
 966 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 967 {
 968         int retval;
 969
 970         BUG_ON(!dqm);
 971
 972         if (lock)
 973                 mutex_lock(&dqm->lock);
 974
 975         retval = destroy_queues_cpsch(dqm, false);
 976         if (retval != 0) {
 977                 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
 978                 goto out;
 979         }
 980
 981         if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
 982                 retval = 0;
 983                 goto out;
 984         }
 985
 986         if (dqm->active_runlist) {
 987                 retval = 0;
 988                 goto out;
 989         }
 990
 991         retval = pm_send_runlist(&dqm->packets, &dqm->queues);
 992         if (retval != 0) {
 993                 pr_err("kfd: failed to execute runlist");
 994                 goto out;
 995         }
 996         dqm->active_runlist = true;
 997
 998 out:
 999         if (lock)
1000                 mutex_unlock(&dqm->lock);
1001         return retval;
1002 }
1003
1004 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1005                                 struct qcm_process_device *qpd,
1006                                 struct queue *q)
1007 {
1008         int retval;
1009         struct mqd_manager *mqd;
1010
1011         BUG_ON(!dqm || !qpd || !q);
1012
1013         retval = 0;
1014
1015         /* remove queue from list to prevent rescheduling after preemption */
1016         mutex_lock(&dqm->lock);
1017         mqd = dqm->ops.get_mqd_manager(dqm,
1018                         get_mqd_type_from_queue_type(q->properties.type));
1019         if (!mqd) {
1020                 retval = -ENOMEM;
1021                 goto failed;
1022         }
1023
1024         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1025                 dqm->sdma_queue_count--;
1026
1027         list_del(&q->list);
1028         if (q->properties.is_active)
1029                 dqm->queue_count--;
1030
1031         execute_queues_cpsch(dqm, false);
1032
1033         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1034
1035         /*
1036          * Unconditionally decrement this counter, regardless of the queue's
1037          * type
1038          */
1039         dqm->total_queue_count--;
1040         pr_debug("Total of %d queues are accountable so far\n",
1041                         dqm->total_queue_count);
1042
1043         mutex_unlock(&dqm->lock);
1044
1045         return 0;
1046
1047 failed:
1048         mutex_unlock(&dqm->lock);
1049         return retval;
1050 }
1051
1052 /*
1053  * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1054  * stay in user mode.
1055  */
1056 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1057 /* APE1 limit is inclusive and 64K aligned. */
1058 #define APE1_LIMIT_ALIGNMENT 0xFFFF
1059
1060 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1061                                    struct qcm_process_device *qpd,
1062                                    enum cache_policy default_policy,
1063                                    enum cache_policy alternate_policy,
1064                                    void __user *alternate_aperture_base,
1065                                    uint64_t alternate_aperture_size)
1066 {
1067         bool retval;
1068
1069         pr_debug("kfd: In func %s\n", __func__);
1070
1071         mutex_lock(&dqm->lock);
1072
1073         if (alternate_aperture_size == 0) {
1074                 /* base > limit disables APE1 */
1075                 qpd->sh_mem_ape1_base = 1;
1076                 qpd->sh_mem_ape1_limit = 0;
1077         } else {
1078                 /*
1079                  * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1080                  *                      SH_MEM_APE1_BASE[31:0], 0x0000 }
1081                  * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1082                  *                      SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1083                  * Verify that the base and size parameters can be
1084                  * represented in this format and convert them.
1085                  * Additionally restrict APE1 to user-mode addresses.
1086                  */
1087
1088                 uint64_t base = (uintptr_t)alternate_aperture_base;
1089                 uint64_t limit = base + alternate_aperture_size - 1;
1090
1091                 if (limit <= base)
1092                         goto out;
1093
1094                 if ((base & APE1_FIXED_BITS_MASK) != 0)
1095                         goto out;
1096
1097                 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
1098                         goto out;
1099
1100                 qpd->sh_mem_ape1_base = base >> 16;
1101                 qpd->sh_mem_ape1_limit = limit >> 16;
1102         }
1103
1104         retval = dqm->ops_asic_specific.set_cache_memory_policy(
1105                         dqm,
1106                         qpd,
1107                         default_policy,
1108                         alternate_policy,
1109                         alternate_aperture_base,
1110                         alternate_aperture_size);
1111
1112         if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1113                 program_sh_mem_settings(dqm, qpd);
1114
1115         pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1116                 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1117                 qpd->sh_mem_ape1_limit);
1118
1119         mutex_unlock(&dqm->lock);
1120         return retval;
1121
1122 out:
1123         mutex_unlock(&dqm->lock);
1124         return false;
1125 }
1126
1127 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1128 {
1129         struct device_queue_manager *dqm;
1130
1131         BUG_ON(!dev);
1132
1133         pr_debug("kfd: loading device queue manager\n");
1134
1135         dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1136         if (!dqm)
1137                 return NULL;
1138
1139         dqm->dev = dev;
1140         switch (sched_policy) {
1141         case KFD_SCHED_POLICY_HWS:
1142         case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1143                 /* initialize dqm for cp scheduling */
1144                 dqm->ops.create_queue = create_queue_cpsch;
1145                 dqm->ops.initialize = initialize_cpsch;
1146                 dqm->ops.start = start_cpsch;
1147                 dqm->ops.stop = stop_cpsch;
1148                 dqm->ops.destroy_queue = destroy_queue_cpsch;
1149                 dqm->ops.update_queue = update_queue;
1150                 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1151                 dqm->ops.register_process = register_process_nocpsch;
1152                 dqm->ops.unregister_process = unregister_process_nocpsch;
1153                 dqm->ops.uninitialize = uninitialize_nocpsch;
1154                 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1155                 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1156                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1157                 break;
1158         case KFD_SCHED_POLICY_NO_HWS:
1159                 /* initialize dqm for no cp scheduling */
1160                 dqm->ops.start = start_nocpsch;
1161                 dqm->ops.stop = stop_nocpsch;
1162                 dqm->ops.create_queue = create_queue_nocpsch;
1163                 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1164                 dqm->ops.update_queue = update_queue;
1165                 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1166                 dqm->ops.register_process = register_process_nocpsch;
1167                 dqm->ops.unregister_process = unregister_process_nocpsch;
1168                 dqm->ops.initialize = initialize_nocpsch;
1169                 dqm->ops.uninitialize = uninitialize_nocpsch;
1170                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1171                 break;
1172         default:
1173                 BUG();
1174                 break;
1175         }
1176
1177         switch (dev->device_info->asic_family) {
1178         case CHIP_CARRIZO:
1179                 device_queue_manager_init_vi(&dqm->ops_asic_specific);
1180                 break;
1181
1182         case CHIP_KAVERI:
1183                 device_queue_manager_init_cik(&dqm->ops_asic_specific);
1184                 break;
1185         }
1186
1187         if (dqm->ops.initialize(dqm) != 0) {
1188                 kfree(dqm);
1189                 return NULL;
1190         }
1191
1192         return dqm;
1193 }
1194
1195 void device_queue_manager_uninit(struct device_queue_manager *dqm)
1196 {
1197         BUG_ON(!dqm);
1198
1199         dqm->ops.uninitialize(dqm);
1200         kfree(dqm);
1201 }