drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 /* Initialize a given run queue struct */
  31 static void init_rq(struct amd_run_queue *rq)
  32 {
  33         INIT_LIST_HEAD(&rq->head.list);
  34         rq->head.belongto_rq = rq;
  35         mutex_init(&rq->lock);
  36         atomic_set(&rq->nr_entity, 0);
  37         rq->current_entity = &rq->head;
  38 }
  39
  40 /* Note: caller must hold the lock or in a atomic context */
  41 static void rq_remove_entity(struct amd_run_queue *rq,
  42                              struct amd_sched_entity *entity)
  43 {
  44         if (rq->current_entity == entity)
  45                 rq->current_entity = list_entry(entity->list.prev,
  46                                                 typeof(*entity), list);
  47         list_del_init(&entity->list);
  48         atomic_dec(&rq->nr_entity);
  49 }
  50
  51 static void rq_add_entity(struct amd_run_queue *rq,
  52                           struct amd_sched_entity *entity)
  53 {
  54         list_add_tail(&entity->list, &rq->head.list);
  55         atomic_inc(&rq->nr_entity);
  56 }
  57
  58 /**
  59  * Select next entity from a specified run queue with round robin policy.
  60  * It could return the same entity as current one if current is the only
  61  * available one in the queue. Return NULL if nothing available.
  62  */
  63 static struct amd_sched_entity *rq_select_entity(struct amd_run_queue *rq)
  64 {
  65         struct amd_sched_entity *p = rq->current_entity;
  66         int i = atomic_read(&rq->nr_entity) + 1; /*real count + dummy head*/
  67         while (i) {
  68                 p = list_entry(p->list.next, typeof(*p), list);
  69                 if (!rq->check_entity_status(p)) {
  70                         rq->current_entity = p;
  71                         break;
  72                 }
  73                 i--;
  74         }
  75         return i ? p : NULL;
  76 }
  77
  78 static bool context_entity_is_waiting(struct amd_context_entity *entity)
  79 {
  80         /* TODO: sync obj for multi-ring synchronization */
  81         return false;
  82 }
  83
  84 static int gpu_entity_check_status(struct amd_sched_entity *entity)
  85 {
  86         struct amd_context_entity *tmp = NULL;
  87
  88         if (entity == &entity->belongto_rq->head)
  89                 return -1;
  90
  91         tmp = container_of(entity, typeof(*tmp), generic_entity);
  92         if (kfifo_is_empty(&tmp->job_queue) ||
  93             context_entity_is_waiting(tmp))
  94                 return -1;
  95
  96         return 0;
  97 }
  98
  99 /**
 100  * Note: This function should only been called inside scheduler main
 101  * function for thread safety, there is no other protection here.
 102  * return ture if scheduler has something ready to run.
 103  *
 104  * For active_hw_rq, there is only one producer(scheduler thread) and
 105  * one consumer(ISR). It should be safe to use this function in scheduler
 106  * main thread to decide whether to continue emit more IBs.
 107 */
 108 static bool is_scheduler_ready(struct amd_gpu_scheduler *sched)
 109 {
 110         return !kfifo_is_full(&sched->active_hw_rq);
 111 }
 112
 113 /**
 114  * Select next entity from the kernel run queue, if not available,
 115  * return null.
 116 */
 117 static struct amd_context_entity *kernel_rq_select_context(
 118         struct amd_gpu_scheduler *sched)
 119 {
 120         struct amd_sched_entity *sched_entity = NULL;
 121         struct amd_context_entity *tmp = NULL;
 122         struct amd_run_queue *rq = &sched->kernel_rq;
 123
 124         mutex_lock(&rq->lock);
 125         sched_entity = rq_select_entity(rq);
 126         if (sched_entity)
 127                 tmp = container_of(sched_entity,
 128                                    typeof(*tmp),
 129                                    generic_entity);
 130         mutex_unlock(&rq->lock);
 131         return tmp;
 132 }
 133
 134 /**
 135  * Select next entity containing real IB submissions
 136 */
 137 static struct amd_context_entity *select_context(
 138         struct amd_gpu_scheduler *sched)
 139 {
 140         struct amd_context_entity *wake_entity = NULL;
 141         struct amd_context_entity *tmp;
 142         struct amd_run_queue *rq;
 143
 144         if (!is_scheduler_ready(sched))
 145                 return NULL;
 146
 147         /* Kernel run queue has higher priority than normal run queue*/
 148         tmp = kernel_rq_select_context(sched);
 149         if (tmp != NULL)
 150                 goto exit;
 151
 152         WARN_ON(offsetof(struct amd_context_entity, generic_entity) != 0);
 153
 154         rq = &sched->sched_rq;
 155         mutex_lock(&rq->lock);
 156         tmp = container_of(rq_select_entity(rq),
 157                            typeof(*tmp), generic_entity);
 158         mutex_unlock(&rq->lock);
 159 exit:
 160         if (sched->current_entity && (sched->current_entity != tmp))
 161                 wake_entity = sched->current_entity;
 162         sched->current_entity = tmp;
 163         if (wake_entity)
 164                 wake_up(&wake_entity->wait_queue);
 165         return tmp;
 166 }
 167
 168 /**
 169  * Init a context entity used by scheduler when submit to HW ring.
 170  *
 171  * @sched       The pointer to the scheduler
 172  * @entity      The pointer to a valid amd_context_entity
 173  * @parent      The parent entity of this amd_context_entity
 174  * @rq          The run queue this entity belongs
 175  * @context_id  The context id for this entity
 176  * @jobs        The max number of jobs in the job queue
 177  *
 178  * return 0 if succeed. negative error code on failure
 179 */
 180 int amd_context_entity_init(struct amd_gpu_scheduler *sched,
 181                             struct amd_context_entity *entity,
 182                             struct amd_sched_entity *parent,
 183                             struct amd_run_queue *rq,
 184                             uint32_t context_id,
 185                             uint32_t jobs)
 186 {
 187         uint64_t seq_ring = 0;
 188
 189         if (!(sched && entity && rq))
 190                 return -EINVAL;
 191
 192         memset(entity, 0, sizeof(struct amd_context_entity));
 193         seq_ring = ((uint64_t)sched->ring_id) << 60;
 194         spin_lock_init(&entity->lock);
 195         entity->generic_entity.belongto_rq = rq;
 196         entity->generic_entity.parent = parent;
 197         entity->scheduler = sched;
 198         init_waitqueue_head(&entity->wait_queue);
 199         init_waitqueue_head(&entity->wait_emit);
 200         if(kfifo_alloc(&entity->job_queue,
 201                        jobs * sizeof(void *),
 202                        GFP_KERNEL))
 203                 return -EINVAL;
 204
 205         spin_lock_init(&entity->queue_lock);
 206         entity->tgid = (context_id == AMD_KERNEL_CONTEXT_ID) ?
 207                 AMD_KERNEL_PROCESS_ID : current->tgid;
 208         entity->context_id = context_id;
 209         atomic64_set(&entity->last_emitted_v_seq, seq_ring);
 210         atomic64_set(&entity->last_queued_v_seq, seq_ring);
 211         atomic64_set(&entity->last_signaled_v_seq, seq_ring);
 212
 213         /* Add the entity to the run queue */
 214         mutex_lock(&rq->lock);
 215         rq_add_entity(rq, &entity->generic_entity);
 216         mutex_unlock(&rq->lock);
 217         return 0;
 218 }
 219
 220 /**
 221  * Query if entity is initialized
 222  *
 223  * @sched       Pointer to scheduler instance
 224  * @entity      The pointer to a valid scheduler entity
 225  *
 226  * return true if entity is initialized, false otherwise
 227 */
 228 static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched,
 229                                           struct amd_context_entity *entity)
 230 {
 231         return entity->scheduler == sched &&
 232                 entity->generic_entity.belongto_rq != NULL;
 233 }
 234
 235 static bool is_context_entity_idle(struct amd_gpu_scheduler *sched,
 236                                    struct amd_context_entity *entity)
 237 {
 238         /**
 239          * Idle means no pending IBs, and the entity is not
 240          * currently being used.
 241         */
 242         barrier();
 243         if ((sched->current_entity != entity) &&
 244             kfifo_is_empty(&entity->job_queue))
 245                 return true;
 246
 247         return false;
 248 }
 249
 250 /**
 251  * Destroy a context entity
 252  *
 253  * @sched       Pointer to scheduler instance
 254  * @entity      The pointer to a valid scheduler entity
 255  *
 256  * return 0 if succeed. negative error code on failure
 257  */
 258 int amd_context_entity_fini(struct amd_gpu_scheduler *sched,
 259                             struct amd_context_entity *entity)
 260 {
 261         int r = 0;
 262         struct amd_run_queue *rq = entity->generic_entity.belongto_rq;
 263
 264         if (!is_context_entity_initialized(sched, entity))
 265                 return 0;
 266
 267         /**
 268          * The client will not queue more IBs during this fini, consume existing
 269          * queued IBs
 270         */
 271         r = wait_event_timeout(
 272                 entity->wait_queue,
 273                 is_context_entity_idle(sched, entity),
 274                 msecs_to_jiffies(AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS)
 275                 ) ?  0 : -1;
 276
 277         if (r) {
 278                 if (entity->is_pending)
 279                         DRM_INFO("Entity %u is in waiting state during fini,\
 280                                 all pending ibs will be canceled.\n",
 281                                  entity->context_id);
 282         }
 283
 284         mutex_lock(&rq->lock);
 285         rq_remove_entity(rq, &entity->generic_entity);
 286         mutex_unlock(&rq->lock);
 287         kfifo_free(&entity->job_queue);
 288         return r;
 289 }
 290
 291 /**
 292  * Submit a normal job to the job queue
 293  *
 294  * @sched       The pointer to the scheduler
 295  * @c_entity    The pointer to amd_context_entity
 296  * @job         The pointer to job required to submit
 297  * return 0 if succeed. -1 if failed.
 298  *        -2 indicate queue is full for this client, client should wait untill
 299  *           scheduler consum some queued command.
 300  *        -1 other fail.
 301 */
 302 int amd_sched_push_job(struct amd_gpu_scheduler *sched,
 303                        struct amd_context_entity *c_entity,
 304                        void *job)
 305 {
 306         while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *),
 307                                    &c_entity->queue_lock) != sizeof(void *)) {
 308                 /**
 309                  * Current context used up all its IB slots
 310                  * wait here, or need to check whether GPU is hung
 311                 */
 312                 schedule();
 313         }
 314
 315         wake_up_interruptible(&sched->wait_queue);
 316         return 0;
 317 }
 318
 319 /**
 320  * Check the virtual sequence number for specified context
 321  *
 322  * @seq         The virtual sequence number to check
 323  * @c_entity    The pointer to a valid amd_context_entity
 324  *
 325  * return 0 if signaled, -1 else.
 326 */
 327 int amd_sched_check_ts(struct amd_context_entity *c_entity, uint64_t seq)
 328 {
 329         return (seq <= atomic64_read(&c_entity->last_signaled_v_seq)) ? 0 : -1;
 330 }
 331
 332 /**
 333  * Wait for a virtual sequence number to be signaled or timeout
 334  *
 335  * @c_entity    The pointer to a valid context entity
 336  * @seq         The virtual sequence number to wait
 337  * @intr        Interruptible or not
 338  * @timeout     Timeout in ms, wait infinitely if <0
 339  * @emit        wait for emit or signal
 340  *
 341  * return =0 signaled ,  <0 failed
 342 */
 343 static int amd_sched_wait(struct amd_context_entity *c_entity,
 344                           uint64_t seq,
 345                           bool intr,
 346                           long timeout,
 347                           bool emit)
 348 {
 349         atomic64_t *v_seq = emit ? &c_entity->last_emitted_v_seq :
 350                 &c_entity->last_signaled_v_seq;
 351         wait_queue_head_t *wait_queue = emit ? &c_entity->wait_emit :
 352                 &c_entity->wait_queue;
 353
 354         if (intr && (timeout < 0)) {
 355                 wait_event_interruptible(
 356                         *wait_queue,
 357                         seq <= atomic64_read(v_seq));
 358                 return 0;
 359         } else if (intr && (timeout >= 0)) {
 360                 wait_event_interruptible_timeout(
 361                         *wait_queue,
 362                         seq <= atomic64_read(v_seq),
 363                         msecs_to_jiffies(timeout));
 364                 return (seq <= atomic64_read(v_seq)) ?
 365                         0 : -1;
 366         } else if (!intr && (timeout < 0)) {
 367                 wait_event(
 368                         *wait_queue,
 369                         seq <= atomic64_read(v_seq));
 370                 return 0;
 371         } else if (!intr && (timeout >= 0)) {
 372                 wait_event_timeout(
 373                         *wait_queue,
 374                         seq <= atomic64_read(v_seq),
 375                         msecs_to_jiffies(timeout));
 376                 return (seq <= atomic64_read(v_seq)) ?
 377                         0 : -1;
 378         }
 379         return 0;
 380 }
 381
 382 int amd_sched_wait_signal(struct amd_context_entity *c_entity,
 383                           uint64_t seq,
 384                           bool intr,
 385                           long timeout)
 386 {
 387         return amd_sched_wait(c_entity, seq, intr, timeout, false);
 388 }
 389
 390 int amd_sched_wait_emit(struct amd_context_entity *c_entity,
 391                         uint64_t seq,
 392                         bool intr,
 393                         long timeout)
 394 {
 395         return amd_sched_wait(c_entity, seq, intr, timeout, true);
 396 }
 397
 398 static int amd_sched_main(void *param)
 399 {
 400         int r;
 401         void *job;
 402         struct sched_param sparam = {.sched_priority = 1};
 403         struct amd_context_entity *c_entity = NULL;
 404         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 405
 406         sched_setscheduler(current, SCHED_FIFO, &sparam);
 407
 408         while (!kthread_should_stop()) {
 409                 wait_event_interruptible(sched->wait_queue,
 410                                          is_scheduler_ready(sched) &&
 411                                          (c_entity = select_context(sched)));
 412                 r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
 413                 if (r != sizeof(void *))
 414                         continue;
 415                 r = sched->ops->prepare_job(sched, c_entity, job);
 416                 if (!r)
 417                         WARN_ON(kfifo_in_spinlocked(
 418                                         &sched->active_hw_rq,
 419                                         &job,
 420                                         sizeof(void *),
 421                                         &sched->queue_lock) != sizeof(void *));
 422                 mutex_lock(&sched->sched_lock);
 423                 sched->ops->run_job(sched, c_entity, job);
 424                 mutex_unlock(&sched->sched_lock);
 425         }
 426         return 0;
 427 }
 428
 429 uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched)
 430 {
 431         return sched->last_handled_seq;
 432 }
 433
 434 /**
 435  * ISR to handle EOP inetrrupts
 436  *
 437  * @sched: gpu scheduler
 438  *
 439 */
 440 void amd_sched_isr(struct amd_gpu_scheduler *sched)
 441 {
 442         int r;
 443         void *job;
 444         r = kfifo_out_spinlocked(&sched->active_hw_rq,
 445                                  &job, sizeof(void *),
 446                                  &sched->queue_lock);
 447
 448         if (r != sizeof(void *))
 449                 job = NULL;
 450
 451         sched->ops->process_job(sched, job);
 452         sched->last_handled_seq++;
 453         wake_up_interruptible(&sched->wait_queue);
 454 }
 455
 456 /**
 457  * Create a gpu scheduler
 458  *
 459  * @device      The device context for this scheduler
 460  * @ops         The backend operations for this scheduler.
 461  * @id          The scheduler is per ring, here is ring id.
 462  * @granularity The minumum ms unit the scheduler will scheduled.
 463  * @preemption  Indicate whether this ring support preemption, 0 is no.
 464  *
 465  * return the pointer to scheduler for success, otherwise return NULL
 466 */
 467 struct amd_gpu_scheduler *amd_sched_create(void *device,
 468                                            struct amd_sched_backend_ops *ops,
 469                                            unsigned ring,
 470                                            unsigned granularity,
 471                                            unsigned preemption)
 472 {
 473         struct amd_gpu_scheduler *sched;
 474         char name[20] = "gpu_sched[0]";
 475
 476         sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
 477         if (!sched)
 478                 return NULL;
 479
 480         sched->device = device;
 481         sched->ops = ops;
 482         sched->granularity = granularity;
 483         sched->ring_id = ring;
 484         sched->preemption = preemption;
 485         sched->last_handled_seq = 0;
 486
 487         snprintf(name, sizeof(name), "gpu_sched[%d]", ring);
 488         mutex_init(&sched->sched_lock);
 489         spin_lock_init(&sched->queue_lock);
 490         init_rq(&sched->sched_rq);
 491         sched->sched_rq.check_entity_status = gpu_entity_check_status;
 492
 493         init_rq(&sched->kernel_rq);
 494         sched->kernel_rq.check_entity_status = gpu_entity_check_status;
 495
 496         init_waitqueue_head(&sched->wait_queue);
 497         if(kfifo_alloc(&sched->active_hw_rq,
 498                        AMD_MAX_ACTIVE_HW_SUBMISSION * sizeof(void *),
 499                        GFP_KERNEL)) {
 500                 kfree(sched);
 501                 return NULL;
 502         }
 503
 504         /* Each scheduler will run on a seperate kernel thread */
 505         sched->thread = kthread_create(amd_sched_main, sched, name);
 506         if (sched->thread) {
 507                 wake_up_process(sched->thread);
 508                 return sched;
 509         }
 510
 511         DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
 512         kfifo_free(&sched->active_hw_rq);
 513         kfree(sched);
 514         return NULL;
 515 }
 516
 517 /**
 518  * Destroy a gpu scheduler
 519  *
 520  * @sched       The pointer to the scheduler
 521  *
 522  * return 0 if succeed. -1 if failed.
 523  */
 524 int amd_sched_destroy(struct amd_gpu_scheduler *sched)
 525 {
 526         kthread_stop(sched->thread);
 527         kfifo_free(&sched->active_hw_rq);
 528         kfree(sched);
 529         return  0;
 530 }
 531