drivers/gpu/arm/mali400/mali/common/mali_scheduler.c

   1 /*
   2  * Copyright (C) 2012-2014 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_scheduler.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_profiling.h"
  15 #include "mali_kernel_utilization.h"
  16 #include "mali_timeline.h"
  17 #include "mali_gp_job.h"
  18 #include "mali_pp_job.h"
  19 #include "mali_executor.h"
  20 #include "mali_group.h"
  21
  22 #if defined(CONFIG_DMA_SHARED_BUFFER)
  23 #include "mali_memory_dma_buf.h"
  24 #endif
  25
  26 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
  27 #include <linux/sched.h>
  28 #include <trace/events/gpu.h>
  29 #endif
  30 /*
  31  * ---------- static defines/constants ----------
  32  */
  33
  34 /*
  35  * If dma_buf with map on demand is used, we defer job deletion and job queue
  36  * if in atomic context, since both might sleep.
  37  */
  38 #if defined(CONFIG_DMA_SHARED_BUFFER)
  39 #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  40 #define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE 1
  41 #define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1
  42 #endif
  43 #endif
  44
  45 /*
  46  * ---------- global variables (exported due to inline functions) ----------
  47  */
  48
  49 /* Lock protecting this module */
  50 _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL;
  51
  52 /* Queue of jobs to be executed on the GP group */
  53 struct mali_scheduler_job_queue job_queue_gp;
  54
  55 /* Queue of PP jobs */
  56 struct mali_scheduler_job_queue job_queue_pp;
  57
  58 _mali_osk_atomic_t mali_job_id_autonumber;
  59 _mali_osk_atomic_t mali_job_cache_order_autonumber;
  60 /*
  61  * ---------- static variables ----------
  62  */
  63
  64 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
  65 static _mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL;
  66 static _mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL;
  67 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue);
  68 #endif
  69
  70 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
  71 static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL;
  72 static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL;
  73 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list);
  74 #endif
  75
  76 /*
  77  * ---------- Forward declaration of static functions ----------
  78  */
  79
  80 static mali_timeline_point mali_scheduler_submit_gp_job(
  81         struct mali_session_data *session, struct mali_gp_job *job);
  82 static mali_timeline_point mali_scheduler_submit_pp_job(
  83         struct mali_session_data *session, struct mali_pp_job *job);
  84
  85 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job);
  86 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job);
  87
  88 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
  89                 mali_bool success);
  90 static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
  91                 u32 num_cores_in_virtual);
  92
  93 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
  94 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job);
  95 static void mali_scheduler_do_pp_job_delete(void *arg);
  96 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
  97
  98 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
  99 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job);
 100 static void mali_scheduler_do_pp_job_queue(void *arg);
 101 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 102
 103 /*
 104  * ---------- Actual implementation ----------
 105  */
 106
 107 _mali_osk_errcode_t mali_scheduler_initialize(void)
 108 {
 109         _mali_osk_atomic_init(&mali_job_id_autonumber, 0);
 110         _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0);
 111
 112         _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri);
 113         _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri);
 114         job_queue_gp.depth = 0;
 115
 116         _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri);
 117         _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri);
 118         job_queue_pp.depth = 0;
 119
 120         mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init(
 121                                           _MALI_OSK_LOCKFLAG_ORDERED,
 122                                           _MALI_OSK_LOCK_ORDER_SCHEDULER);
 123         if (NULL == mali_scheduler_lock_obj) {
 124                 mali_scheduler_terminate();
 125         }
 126
 127 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
 128         scheduler_wq_pp_job_delete = _mali_osk_wq_create_work(
 129                                              mali_scheduler_do_pp_job_delete, NULL);
 130         if (NULL == scheduler_wq_pp_job_delete) {
 131                 mali_scheduler_terminate();
 132                 return _MALI_OSK_ERR_FAULT;
 133         }
 134
 135         scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init(
 136                                                _MALI_OSK_LOCKFLAG_ORDERED,
 137                                                _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 138         if (NULL == scheduler_pp_job_delete_lock) {
 139                 mali_scheduler_terminate();
 140                 return _MALI_OSK_ERR_FAULT;
 141         }
 142 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
 143
 144 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 145         scheduler_wq_pp_job_queue = _mali_osk_wq_create_work(
 146                                             mali_scheduler_do_pp_job_queue, NULL);
 147         if (NULL == scheduler_wq_pp_job_queue) {
 148                 mali_scheduler_terminate();
 149                 return _MALI_OSK_ERR_FAULT;
 150         }
 151
 152         scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init(
 153                                               _MALI_OSK_LOCKFLAG_ORDERED,
 154                                               _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 155         if (NULL == scheduler_pp_job_queue_lock) {
 156                 mali_scheduler_terminate();
 157                 return _MALI_OSK_ERR_FAULT;
 158         }
 159 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 160
 161         return _MALI_OSK_ERR_OK;
 162 }
 163
 164 void mali_scheduler_terminate(void)
 165 {
 166 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 167         if (NULL != scheduler_pp_job_queue_lock) {
 168                 _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock);
 169                 scheduler_pp_job_queue_lock = NULL;
 170         }
 171
 172         if (NULL != scheduler_wq_pp_job_queue) {
 173                 _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue);
 174                 scheduler_wq_pp_job_queue = NULL;
 175         }
 176 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 177
 178 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
 179         if (NULL != scheduler_pp_job_delete_lock) {
 180                 _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock);
 181                 scheduler_pp_job_delete_lock = NULL;
 182         }
 183
 184         if (NULL != scheduler_wq_pp_job_delete) {
 185                 _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete);
 186                 scheduler_wq_pp_job_delete = NULL;
 187         }
 188 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
 189
 190         if (NULL != mali_scheduler_lock_obj) {
 191                 _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj);
 192                 mali_scheduler_lock_obj = NULL;
 193         }
 194
 195         _mali_osk_atomic_term(&mali_job_cache_order_autonumber);
 196         _mali_osk_atomic_term(&mali_job_id_autonumber);
 197 }
 198
 199 u32 mali_scheduler_job_physical_head_count(void)
 200 {
 201         /*
 202          * Count how many physical sub jobs are present from the head of queue
 203          * until the first virtual job is present.
 204          * Early out when we have reached maximum number of PP cores (8)
 205          */
 206         u32 count = 0;
 207         struct mali_pp_job *job;
 208         struct mali_pp_job *temp;
 209
 210         /* Check for partially started normal pri jobs */
 211         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 212                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 213
 214                 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 215                                            struct mali_pp_job, list);
 216
 217                 MALI_DEBUG_ASSERT_POINTER(job);
 218
 219                 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
 220                         /*
 221                          * Remember; virtual jobs can't be queued and started
 222                          * at the same time, so this must be a physical job
 223                          */
 224                         count += mali_pp_job_unstarted_sub_job_count(job);
 225                         if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 226                                 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 227                         }
 228                 }
 229         }
 230
 231         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
 232                                     struct mali_pp_job, list) {
 233                 if (MALI_FALSE == mali_pp_job_is_virtual(job)) {
 234                         count += mali_pp_job_unstarted_sub_job_count(job);
 235                         if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 236                                 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 237                         }
 238                 } else {
 239                         /* Came across a virtual job, so stop counting */
 240                         return count;
 241                 }
 242         }
 243
 244         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
 245                                     struct mali_pp_job, list) {
 246                 if (MALI_FALSE == mali_pp_job_is_virtual(job)) {
 247                         /* any partially started is already counted */
 248                         if (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
 249                                 count += mali_pp_job_unstarted_sub_job_count(job);
 250                                 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <=
 251                                     count) {
 252                                         return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 253                                 }
 254                         }
 255                 } else {
 256                         /* Came across a virtual job, so stop counting */
 257                         return count;
 258                 }
 259         }
 260
 261         return count;
 262 }
 263
 264 mali_bool mali_scheduler_job_next_is_virtual(void)
 265 {
 266         struct mali_pp_job *job;
 267
 268         job = mali_scheduler_job_pp_virtual_peek();
 269         if (NULL != job) {
 270                 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
 271
 272                 return MALI_TRUE;
 273         }
 274
 275         return MALI_FALSE;
 276 }
 277
 278 struct mali_gp_job *mali_scheduler_job_gp_get(void)
 279 {
 280         _mali_osk_list_t *queue;
 281         struct mali_gp_job *job = NULL;
 282
 283         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 284         MALI_DEBUG_ASSERT(0 < job_queue_gp.depth);
 285
 286         if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
 287                 queue = &job_queue_gp.high_pri;
 288         } else {
 289                 queue = &job_queue_gp.normal_pri;
 290                 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue));
 291         }
 292
 293         job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list);
 294
 295         MALI_DEBUG_ASSERT_POINTER(job);
 296
 297         mali_gp_job_list_remove(job);
 298         job_queue_gp.depth--;
 299
 300         return job;
 301 }
 302
 303 struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void)
 304 {
 305         struct mali_pp_job *job = NULL;
 306         struct mali_pp_job *tmp_job = NULL;
 307
 308         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 309
 310         /*
 311          * For PP jobs we favour partially started jobs in normal
 312          * priority queue over unstarted jobs in high priority queue
 313          */
 314
 315         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 316                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 317
 318                 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 319                                                struct mali_pp_job, list);
 320                 MALI_DEBUG_ASSERT(NULL != tmp_job);
 321
 322                 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
 323                         job = tmp_job;
 324                 }
 325         }
 326
 327         if (NULL == job ||
 328             MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
 329                 /*
 330                  * There isn't a partially started job in normal queue, so
 331                  * look in high priority queue.
 332                  */
 333                 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
 334                         MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 335
 336                         tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
 337                                                        struct mali_pp_job, list);
 338                         MALI_DEBUG_ASSERT(NULL != tmp_job);
 339
 340                         if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
 341                                 job = tmp_job;
 342                         }
 343                 }
 344         }
 345
 346         return job;
 347 }
 348
 349 struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void)
 350 {
 351         struct mali_pp_job *job = NULL;
 352         struct mali_pp_job *tmp_job = NULL;
 353
 354         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 355
 356         if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
 357                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 358
 359                 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
 360                                                struct mali_pp_job, list);
 361
 362                 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
 363                         job = tmp_job;
 364                 }
 365         }
 366
 367         if (NULL == job) {
 368                 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 369                         MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 370
 371                         tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 372                                                        struct mali_pp_job, list);
 373
 374                         if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
 375                                 job = tmp_job;
 376                         }
 377                 }
 378         }
 379
 380         return job;
 381 }
 382
 383 struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job)
 384 {
 385         struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek();
 386
 387         MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job));
 388
 389         if (NULL != job) {
 390                 *sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
 391
 392                 mali_pp_job_mark_sub_job_started(job, *sub_job);
 393                 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) {
 394                         /* Remove from queue when last sub job has been retrieved */
 395                         mali_pp_job_list_remove(job);
 396                 }
 397
 398                 job_queue_pp.depth--;
 399
 400                 /*
 401                  * Job about to start so it is no longer be
 402                  * possible to discard WB
 403                  */
 404                 mali_pp_job_fb_lookup_remove(job);
 405         }
 406
 407         return job;
 408 }
 409
 410 struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void)
 411 {
 412         struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek();
 413
 414         MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job));
 415
 416         if (NULL != job) {
 417                 MALI_DEBUG_ASSERT(0 ==
 418                                   mali_pp_job_get_first_unstarted_sub_job(job));
 419                 MALI_DEBUG_ASSERT(1 ==
 420                                   mali_pp_job_get_sub_job_count(job));
 421
 422                 mali_pp_job_mark_sub_job_started(job, 0);
 423
 424                 mali_pp_job_list_remove(job);
 425
 426                 job_queue_pp.depth--;
 427
 428                 /*
 429                  * Job about to start so it is no longer be
 430                  * possible to discard WB
 431                  */
 432                 mali_pp_job_fb_lookup_remove(job);
 433         }
 434
 435         return job;
 436 }
 437
 438 mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job)
 439 {
 440         MALI_DEBUG_ASSERT_POINTER(job);
 441
 442         MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n",
 443                              mali_gp_job_get_id(job), job));
 444
 445         mali_scheduler_lock();
 446
 447         if (!mali_scheduler_queue_gp_job(job)) {
 448                 /* Failed to enqueue job, release job (with error) */
 449
 450                 mali_scheduler_unlock();
 451
 452                 mali_timeline_tracker_release(mali_gp_job_get_tracker(job));
 453                 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
 454
 455                 /* This will notify user space and close the job object */
 456                 mali_scheduler_complete_gp_job(job, MALI_FALSE,
 457                                                MALI_TRUE, MALI_FALSE);
 458
 459                 return MALI_SCHEDULER_MASK_EMPTY;
 460         }
 461
 462         mali_scheduler_unlock();
 463
 464         return MALI_SCHEDULER_MASK_GP;
 465 }
 466
 467 mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job)
 468 {
 469         MALI_DEBUG_ASSERT_POINTER(job);
 470
 471         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n",
 472                              mali_pp_job_get_id(job), job));
 473
 474         if (MALI_TRUE == mali_timeline_tracker_activation_error(
 475                     mali_pp_job_get_tracker(job))) {
 476                 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n",
 477                                      mali_pp_job_get_id(job), job));
 478
 479                 mali_scheduler_lock();
 480                 mali_pp_job_fb_lookup_remove(job);
 481                 mali_pp_job_mark_unstarted_failed(job);
 482                 mali_scheduler_unlock();
 483
 484                 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
 485
 486                 /* This will notify user space and close the job object */
 487                 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
 488
 489                 return MALI_SCHEDULER_MASK_EMPTY;
 490         }
 491
 492 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 493         if (mali_pp_job_needs_dma_buf_mapping(job)) {
 494                 mali_scheduler_deferred_pp_job_queue(job);
 495                 return MALI_SCHEDULER_MASK_EMPTY;
 496         }
 497 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 498
 499         mali_scheduler_lock();
 500
 501         if (!mali_scheduler_queue_pp_job(job)) {
 502                 /* Failed to enqueue job, release job (with error) */
 503                 mali_pp_job_fb_lookup_remove(job);
 504                 mali_pp_job_mark_unstarted_failed(job);
 505                 mali_scheduler_unlock();
 506
 507                 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
 508
 509                 /* This will notify user space and close the job object */
 510                 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
 511
 512                 return MALI_SCHEDULER_MASK_EMPTY;
 513         }
 514
 515         mali_scheduler_unlock();
 516         return MALI_SCHEDULER_MASK_PP;
 517 }
 518
 519 void mali_scheduler_complete_gp_job(struct mali_gp_job *job,
 520                                     mali_bool success,
 521                                     mali_bool user_notification,
 522                                     mali_bool dequeued)
 523 {
 524         if (user_notification) {
 525                 mali_scheduler_return_gp_job_to_user(job, success);
 526         }
 527
 528         if (dequeued) {
 529                 _mali_osk_pm_dev_ref_put();
 530
 531                 if (mali_utilization_enabled()) {
 532                         mali_utilization_gp_end();
 533                 }
 534         }
 535
 536         mali_gp_job_delete(job);
 537 }
 538
 539 void mali_scheduler_complete_pp_job(struct mali_pp_job *job,
 540                                     u32 num_cores_in_virtual,
 541                                     mali_bool user_notification,
 542                                     mali_bool dequeued)
 543 {
 544         if (user_notification) {
 545                 mali_scheduler_return_pp_job_to_user(job,
 546                                                      num_cores_in_virtual);
 547         }
 548
 549         if (dequeued) {
 550 #if defined(CONFIG_MALI_DVFS)
 551                 if (mali_pp_job_is_window_surface(job)) {
 552                         struct mali_session_data *session;
 553                         session = mali_pp_job_get_session(job);
 554                         mali_session_inc_num_window_jobs(session);
 555                 }
 556 #endif
 557
 558                 _mali_osk_pm_dev_ref_put();
 559
 560                 if (mali_utilization_enabled()) {
 561                         mali_utilization_pp_end();
 562                 }
 563         }
 564
 565 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
 566         /*
 567          * The deletion of the job object (releasing sync refs etc)
 568          * must be done in a different context
 569          */
 570         mali_scheduler_deferred_pp_job_delete(job);
 571 #else
 572         /* no use cases need this in this configuration */
 573         mali_pp_job_delete(job);
 574 #endif
 575 }
 576
 577 void mali_scheduler_abort_session(struct mali_session_data *session)
 578 {
 579         struct mali_gp_job *gp_job;
 580         struct mali_gp_job *gp_tmp;
 581         struct mali_pp_job *pp_job;
 582         struct mali_pp_job *pp_tmp;
 583         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp);
 584         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp);
 585
 586         MALI_DEBUG_ASSERT_POINTER(session);
 587         MALI_DEBUG_ASSERT(session->is_aborting);
 588
 589         MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n",
 590                              session));
 591
 592         mali_scheduler_lock();
 593
 594         /* Remove from GP normal priority queue */
 595         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri,
 596                                     struct mali_gp_job, list) {
 597                 if (mali_gp_job_get_session(gp_job) == session) {
 598                         mali_gp_job_list_move(gp_job, &removed_jobs_gp);
 599                         job_queue_gp.depth--;
 600                 }
 601         }
 602
 603         /* Remove from GP high priority queue */
 604         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri,
 605                                     struct mali_gp_job, list) {
 606                 if (mali_gp_job_get_session(gp_job) == session) {
 607                         mali_gp_job_list_move(gp_job, &removed_jobs_gp);
 608                         job_queue_gp.depth--;
 609                 }
 610         }
 611
 612         /* Remove from PP normal priority queue */
 613         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
 614                                     &job_queue_pp.normal_pri,
 615                                     struct mali_pp_job, list) {
 616                 if (mali_pp_job_get_session(pp_job) == session) {
 617                         mali_pp_job_fb_lookup_remove(pp_job);
 618
 619                         job_queue_pp.depth -=
 620                                 mali_pp_job_unstarted_sub_job_count(
 621                                         pp_job);
 622                         mali_pp_job_mark_unstarted_failed(pp_job);
 623
 624                         if (mali_pp_job_is_complete(pp_job)) {
 625                                 mali_pp_job_list_move(pp_job,
 626                                                       &removed_jobs_pp);
 627                         }
 628                 }
 629         }
 630
 631         /* Remove from PP high priority queue */
 632         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
 633                                     &job_queue_pp.high_pri,
 634                                     struct mali_pp_job, list) {
 635                 if (mali_pp_job_get_session(pp_job) == session) {
 636                         mali_pp_job_fb_lookup_remove(pp_job);
 637
 638                         job_queue_pp.depth -=
 639                                 mali_pp_job_unstarted_sub_job_count(
 640                                         pp_job);
 641                         mali_pp_job_mark_unstarted_failed(pp_job);
 642
 643                         if (mali_pp_job_is_complete(pp_job)) {
 644                                 mali_pp_job_list_move(pp_job,
 645                                                       &removed_jobs_pp);
 646                         }
 647                 }
 648         }
 649
 650         /*
 651          * Release scheduler lock so we can release trackers
 652          * (which will potentially queue new jobs)
 653          */
 654         mali_scheduler_unlock();
 655
 656         /* Release and complete all (non-running) found GP jobs  */
 657         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp,
 658                                     struct mali_gp_job, list) {
 659                 mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job));
 660                 mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE);
 661                 _mali_osk_list_delinit(&gp_job->list);
 662                 mali_scheduler_complete_gp_job(gp_job,
 663                                                MALI_FALSE, MALI_FALSE, MALI_TRUE);
 664         }
 665
 666         /* Release and complete non-running PP jobs */
 667         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp,
 668                                     struct mali_pp_job, list) {
 669                 mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job));
 670                 _mali_osk_list_delinit(&pp_job->list);
 671                 mali_scheduler_complete_pp_job(pp_job, 0,
 672                                                MALI_FALSE, MALI_TRUE);
 673         }
 674 }
 675
 676 _mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx,
 677                 _mali_uk_gp_start_job_s *uargs)
 678 {
 679         struct mali_session_data *session;
 680         struct mali_gp_job *job;
 681         mali_timeline_point point;
 682         u32 __user *point_ptr = NULL;
 683
 684         MALI_DEBUG_ASSERT_POINTER(uargs);
 685         MALI_DEBUG_ASSERT_POINTER(ctx);
 686
 687         session = (struct mali_session_data *)(uintptr_t)ctx;
 688
 689         job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(),
 690                                  NULL);
 691         if (NULL == job) {
 692                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 693                 return _MALI_OSK_ERR_NOMEM;
 694         }
 695
 696         point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job);
 697
 698         point = mali_scheduler_submit_gp_job(session, job);
 699
 700         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 701                 /*
 702                  * Let user space know that something failed
 703                  * after the job was started.
 704                  */
 705                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 706         }
 707
 708         return _MALI_OSK_ERR_OK;
 709 }
 710
 711 _mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx,
 712                 _mali_uk_pp_start_job_s *uargs)
 713 {
 714         struct mali_session_data *session;
 715         struct mali_pp_job *job;
 716         mali_timeline_point point;
 717         u32 __user *point_ptr = NULL;
 718
 719         MALI_DEBUG_ASSERT_POINTER(uargs);
 720         MALI_DEBUG_ASSERT_POINTER(ctx);
 721
 722         session = (struct mali_session_data *)(uintptr_t)ctx;
 723
 724         job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
 725         if (NULL == job) {
 726                 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
 727                 return _MALI_OSK_ERR_NOMEM;
 728         }
 729
 730         point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job);
 731
 732         point = mali_scheduler_submit_pp_job(session, job);
 733         job = NULL;
 734
 735         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 736                 /*
 737                  * Let user space know that something failed
 738                  * after the job was started.
 739                  */
 740                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 741         }
 742
 743         return _MALI_OSK_ERR_OK;
 744 }
 745
 746 _mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx,
 747                 _mali_uk_pp_and_gp_start_job_s *uargs)
 748 {
 749         struct mali_session_data *session;
 750         _mali_uk_pp_and_gp_start_job_s kargs;
 751         struct mali_pp_job *pp_job;
 752         struct mali_gp_job *gp_job;
 753         u32 __user *point_ptr = NULL;
 754         mali_timeline_point point;
 755         _mali_uk_pp_start_job_s __user *pp_args;
 756         _mali_uk_gp_start_job_s __user *gp_args;
 757
 758         MALI_DEBUG_ASSERT_POINTER(ctx);
 759         MALI_DEBUG_ASSERT_POINTER(uargs);
 760
 761         session = (struct mali_session_data *) ctx;
 762
 763         if (0 != _mali_osk_copy_from_user(&kargs, uargs,
 764                                           sizeof(_mali_uk_pp_and_gp_start_job_s))) {
 765                 return _MALI_OSK_ERR_NOMEM;
 766         }
 767
 768         pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args;
 769         gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args;
 770
 771         pp_job = mali_pp_job_create(session, pp_args,
 772                                     mali_scheduler_get_new_id());
 773         if (NULL == pp_job) {
 774                 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
 775                 return _MALI_OSK_ERR_NOMEM;
 776         }
 777
 778         gp_job = mali_gp_job_create(session, gp_args,
 779                                     mali_scheduler_get_new_id(),
 780                                     mali_pp_job_get_tracker(pp_job));
 781         if (NULL == gp_job) {
 782                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 783                 mali_pp_job_delete(pp_job);
 784                 return _MALI_OSK_ERR_NOMEM;
 785         }
 786
 787         point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job);
 788
 789         /* Submit GP job. */
 790         mali_scheduler_submit_gp_job(session, gp_job);
 791         gp_job = NULL;
 792
 793         /* Submit PP job. */
 794         point = mali_scheduler_submit_pp_job(session, pp_job);
 795         pp_job = NULL;
 796
 797         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 798                 /*
 799                  * Let user space know that something failed
 800                  * after the jobs were started.
 801                  */
 802                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 803         }
 804
 805         return _MALI_OSK_ERR_OK;
 806 }
 807
 808 void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
 809 {
 810         struct mali_session_data *session;
 811         struct mali_pp_job *job;
 812         struct mali_pp_job *tmp;
 813         u32 fb_lookup_id;
 814
 815         MALI_DEBUG_ASSERT_POINTER(args);
 816         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
 817
 818         session = (struct mali_session_data *)(uintptr_t)args->ctx;
 819
 820         fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK;
 821
 822         mali_scheduler_lock();
 823
 824         /* Iterate over all jobs for given frame builder_id. */
 825         _MALI_OSK_LIST_FOREACHENTRY(job, tmp,
 826                                     &session->pp_job_fb_lookup_list[fb_lookup_id],
 827                                     struct mali_pp_job, session_fb_lookup_list) {
 828                 MALI_DEBUG_CODE(u32 disable_mask = 0);
 829
 830                 if (mali_pp_job_get_frame_builder_id(job) !=
 831                     (u32) args->fb_id) {
 832                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n"));
 833                         continue;
 834                 }
 835
 836                 MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3));
 837
 838                 if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) {
 839                         MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1));
 840                         mali_pp_job_disable_wb0(job);
 841                 }
 842
 843                 if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) {
 844                         MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2));
 845                         mali_pp_job_disable_wb1(job);
 846                 }
 847
 848                 if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) {
 849                         MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3));
 850                         mali_pp_job_disable_wb2(job);
 851                 }
 852                 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n",
 853                                      disable_mask));
 854         }
 855
 856         mali_scheduler_unlock();
 857 }
 858
 859 #if MALI_STATE_TRACKING
 860 u32 mali_scheduler_dump_state(char *buf, u32 size)
 861 {
 862         int n = 0;
 863
 864         n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n");
 865         n += _mali_osk_snprintf(buf + n, size - n,
 866                                 "\tQueue depth: %u\n", job_queue_gp.depth);
 867         n += _mali_osk_snprintf(buf + n, size - n,
 868                                 "\tNormal priority queue is %s\n",
 869                                 _mali_osk_list_empty(&job_queue_gp.normal_pri) ?
 870                                 "empty" : "not empty");
 871         n += _mali_osk_snprintf(buf + n, size - n,
 872                                 "\tHigh priority queue is %s\n",
 873                                 _mali_osk_list_empty(&job_queue_gp.high_pri) ?
 874                                 "empty" : "not empty");
 875
 876         n += _mali_osk_snprintf(buf + n, size - n,
 877                                 "PP queues\n");
 878         n += _mali_osk_snprintf(buf + n, size - n,
 879                                 "\tQueue depth: %u\n", job_queue_pp.depth);
 880         n += _mali_osk_snprintf(buf + n, size - n,
 881                                 "\tNormal priority queue is %s\n",
 882                                 _mali_osk_list_empty(&job_queue_pp.normal_pri)
 883                                 ? "empty" : "not empty");
 884         n += _mali_osk_snprintf(buf + n, size - n,
 885                                 "\tHigh priority queue is %s\n",
 886                                 _mali_osk_list_empty(&job_queue_pp.high_pri)
 887                                 ? "empty" : "not empty");
 888
 889         n += _mali_osk_snprintf(buf + n, size - n, "\n");
 890
 891         return n;
 892 }
 893 #endif
 894
 895 /*
 896  * ---------- Implementation of static functions ----------
 897  */
 898
 899 static mali_timeline_point mali_scheduler_submit_gp_job(
 900         struct mali_session_data *session, struct mali_gp_job *job)
 901 {
 902         mali_timeline_point point;
 903
 904         MALI_DEBUG_ASSERT_POINTER(session);
 905         MALI_DEBUG_ASSERT_POINTER(job);
 906
 907         /* Add job to Timeline system. */
 908         point = mali_timeline_system_add_tracker(session->timeline_system,
 909                         mali_gp_job_get_tracker(job), MALI_TIMELINE_GP);
 910
 911         return point;
 912 }
 913
 914 static mali_timeline_point mali_scheduler_submit_pp_job(
 915         struct mali_session_data *session, struct mali_pp_job *job)
 916 {
 917         mali_timeline_point point;
 918
 919         MALI_DEBUG_ASSERT_POINTER(session);
 920         MALI_DEBUG_ASSERT_POINTER(job);
 921
 922         mali_scheduler_lock();
 923         /*
 924          * Adding job to the lookup list used to quickly discard
 925          * writeback units of queued jobs.
 926          */
 927         mali_pp_job_fb_lookup_add(job);
 928         mali_scheduler_unlock();
 929
 930         /* Add job to Timeline system. */
 931         point = mali_timeline_system_add_tracker(session->timeline_system,
 932                         mali_pp_job_get_tracker(job), MALI_TIMELINE_PP);
 933
 934         return point;
 935 }
 936
 937 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job)
 938 {
 939         struct mali_session_data *session;
 940         _mali_osk_list_t *queue;
 941
 942         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
 943         MALI_DEBUG_ASSERT_POINTER(job);
 944
 945         session = mali_gp_job_get_session(job);
 946         MALI_DEBUG_ASSERT_POINTER(session);
 947
 948         if (unlikely(session->is_aborting)) {
 949                 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
 950                                      mali_gp_job_get_id(job), job));
 951                 return MALI_FALSE; /* job not queued */
 952         }
 953
 954         mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
 955
 956         /* Determine which queue the job should be added to. */
 957         if (session->use_high_priority_job_queue) {
 958                 queue = &job_queue_gp.high_pri;
 959         } else {
 960                 queue = &job_queue_gp.normal_pri;
 961         }
 962
 963         job_queue_gp.depth += 1;
 964
 965         /* Add job to queue (mali_gp_job_queue_add find correct place). */
 966         mali_gp_job_list_add(job, queue);
 967
 968         /*
 969          * We hold a PM reference for every job we hold queued (and running)
 970          * It is important that we take this reference after job has been
 971          * added the the queue so that any runtime resume could schedule this
 972          * job right there and then.
 973          */
 974         _mali_osk_pm_dev_ref_get_async();
 975
 976         if (mali_utilization_enabled()) {
 977                 /*
 978                  * We cheat a little bit by counting the GP as busy from the
 979                  * time a GP job is queued. This will be fine because we only
 980                  * loose the tiny idle gap between jobs, but we will instead
 981                  * get less utilization work to do (less locks taken)
 982                  */
 983                 mali_utilization_gp_start();
 984         }
 985
 986         /* Add profiling events for job enqueued */
 987         _mali_osk_profiling_add_event(
 988                 MALI_PROFILING_EVENT_TYPE_SINGLE |
 989                 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
 990                 MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE,
 991                 mali_gp_job_get_pid(job),
 992                 mali_gp_job_get_tid(job),
 993                 mali_gp_job_get_frame_builder_id(job),
 994                 mali_gp_job_get_flush_id(job),
 995                 0);
 996
 997 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
 998         trace_gpu_job_enqueue(mali_gp_job_get_tid(job),
 999                               mali_gp_job_get_id(job), "GP");
1000 #endif
1001
1002         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n",
1003                              mali_gp_job_get_id(job), job));
1004
1005         return MALI_TRUE; /* job queued */
1006 }
1007
1008 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job)
1009 {
1010         struct mali_session_data *session;
1011         _mali_osk_list_t *queue = NULL;
1012
1013         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
1014         MALI_DEBUG_ASSERT_POINTER(job);
1015
1016         session = mali_pp_job_get_session(job);
1017         MALI_DEBUG_ASSERT_POINTER(session);
1018
1019         if (unlikely(session->is_aborting)) {
1020                 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
1021                                      mali_pp_job_get_id(job), job));
1022                 return MALI_FALSE; /* job not queued */
1023         }
1024
1025         mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1026
1027         if (session->use_high_priority_job_queue) {
1028                 queue = &job_queue_pp.high_pri;
1029         } else {
1030                 queue = &job_queue_pp.normal_pri;
1031         }
1032
1033         job_queue_pp.depth +=
1034                 mali_pp_job_get_sub_job_count(job);
1035
1036         /* Add job to queue (mali_gp_job_queue_add find correct place). */
1037         mali_pp_job_list_add(job, queue);
1038
1039         /*
1040          * We hold a PM reference for every job we hold queued (and running)
1041          * It is important that we take this reference after job has been
1042          * added the the queue so that any runtime resume could schedule this
1043          * job right there and then.
1044          */
1045         _mali_osk_pm_dev_ref_get_async();
1046
1047         if (mali_utilization_enabled()) {
1048                 /*
1049                  * We cheat a little bit by counting the PP as busy from the
1050                  * time a PP job is queued. This will be fine because we only
1051                  * loose the tiny idle gap between jobs, but we will instead
1052                  * get less utilization work to do (less locks taken)
1053                  */
1054                 mali_utilization_pp_start();
1055         }
1056
1057         /* Add profiling events for job enqueued */
1058
1059         _mali_osk_profiling_add_event(
1060                 MALI_PROFILING_EVENT_TYPE_SINGLE |
1061                 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1062                 MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE,
1063                 mali_pp_job_get_pid(job),
1064                 mali_pp_job_get_tid(job),
1065                 mali_pp_job_get_frame_builder_id(job),
1066                 mali_pp_job_get_flush_id(job),
1067                 0);
1068
1069 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1070         trace_gpu_job_enqueue(mali_pp_job_get_tid(job),
1071                               mali_pp_job_get_id(job), "PP");
1072 #endif
1073
1074         MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n",
1075                              mali_pp_job_is_virtual(job)
1076                              ? "Virtual" : "Physical",
1077                              mali_pp_job_get_id(job), job,
1078                              mali_pp_job_get_sub_job_count(job)));
1079
1080         return MALI_TRUE; /* job queued */
1081 }
1082
1083 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
1084                 mali_bool success)
1085 {
1086         _mali_uk_gp_job_finished_s *jobres;
1087         struct mali_session_data *session;
1088         _mali_osk_notification_t *notification;
1089
1090         MALI_DEBUG_ASSERT_POINTER(job);
1091
1092         session = mali_gp_job_get_session(job);
1093         MALI_DEBUG_ASSERT_POINTER(session);
1094
1095         notification = mali_gp_job_get_finished_notification(job);
1096         MALI_DEBUG_ASSERT_POINTER(notification);
1097
1098         jobres = notification->result_buffer;
1099         MALI_DEBUG_ASSERT_POINTER(jobres);
1100
1101         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1102         if (MALI_TRUE == success) {
1103                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1104         } else {
1105                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1106         }
1107         jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
1108         jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
1109         jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
1110
1111         mali_session_send_notification(session, notification);
1112 }
1113
1114 static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
1115                 u32 num_cores_in_virtual)
1116 {
1117         u32 i;
1118         u32 num_counters_to_copy;
1119         _mali_uk_pp_job_finished_s *jobres;
1120         struct mali_session_data *session;
1121         _mali_osk_notification_t *notification;
1122
1123         if (MALI_TRUE == mali_pp_job_use_no_notification(job)) {
1124                 return;
1125         }
1126
1127         MALI_DEBUG_ASSERT_POINTER(job);
1128
1129         session = mali_pp_job_get_session(job);
1130         MALI_DEBUG_ASSERT_POINTER(session);
1131
1132         notification = mali_pp_job_get_finished_notification(job);
1133         MALI_DEBUG_ASSERT_POINTER(notification);
1134
1135         jobres = notification->result_buffer;
1136         MALI_DEBUG_ASSERT_POINTER(jobres);
1137
1138         jobres->user_job_ptr = mali_pp_job_get_user_id(job);
1139         if (MALI_TRUE == mali_pp_job_was_success(job)) {
1140                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1141         } else {
1142                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1143         }
1144
1145         if (mali_pp_job_is_virtual(job)) {
1146                 num_counters_to_copy = num_cores_in_virtual;
1147         } else {
1148                 num_counters_to_copy = mali_pp_job_get_sub_job_count(job);
1149         }
1150
1151         for (i = 0; i < num_counters_to_copy; i++) {
1152                 jobres->perf_counter0[i] =
1153                         mali_pp_job_get_perf_counter_value0(job, i);
1154                 jobres->perf_counter1[i] =
1155                         mali_pp_job_get_perf_counter_value1(job, i);
1156                 jobres->perf_counter_src0 =
1157                         mali_pp_job_get_pp_counter_global_src0();
1158                 jobres->perf_counter_src1 =
1159                         mali_pp_job_get_pp_counter_global_src1();
1160         }
1161
1162         mali_session_send_notification(session, notification);
1163 }
1164
1165 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE)
1166
1167 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job)
1168 {
1169         MALI_DEBUG_ASSERT_POINTER(job);
1170
1171         _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1172         mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue);
1173         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1174
1175         _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete);
1176 }
1177
1178 static void mali_scheduler_do_pp_job_delete(void *arg)
1179 {
1180         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1181         struct mali_pp_job *job;
1182         struct mali_pp_job *tmp;
1183
1184         MALI_IGNORE(arg);
1185
1186         /*
1187          * Quickly "unhook" the jobs pending to be deleted, so we can release
1188          * the lock before we start deleting the job objects
1189          * (without any locks held)
1190          */
1191         _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1192         _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list);
1193         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1194
1195         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1196                                     struct mali_pp_job, list) {
1197
1198                 _mali_osk_list_delinit(&job->list);
1199                 mali_pp_job_delete(job); /* delete the job object itself */
1200         }
1201 }
1202
1203 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */
1204
1205 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
1206
1207 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job)
1208 {
1209         MALI_DEBUG_ASSERT_POINTER(job);
1210
1211         _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1212         mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list);
1213         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1214
1215         _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue);
1216 }
1217
1218 static void mali_scheduler_do_pp_job_queue(void *arg)
1219 {
1220         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1221         struct mali_pp_job *job;
1222         struct mali_pp_job *tmp;
1223         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1224
1225         MALI_IGNORE(arg);
1226
1227         /*
1228          * Quickly "unhook" the jobs pending to be queued, so we can release
1229          * the lock before we start queueing the job objects
1230          * (without any locks held)
1231          */
1232         _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1233         _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list);
1234         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1235
1236         /* First loop through all jobs and do the pre-work (no locks needed) */
1237         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1238                                     struct mali_pp_job, list) {
1239                 if (mali_pp_job_needs_dma_buf_mapping(job)) {
1240                         /*
1241                          * This operation could fail, but we continue anyway,
1242                          * because the worst that could happen is that this
1243                          * job will fail due to a Mali page fault.
1244                          */
1245                         mali_dma_buf_map_job(job);
1246                 }
1247         }
1248
1249         mali_scheduler_lock();
1250
1251         /* Then loop through all jobs again to queue them (lock needed) */
1252         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1253                                     struct mali_pp_job, list) {
1254
1255                 /* Remove from scheduler_pp_job_queue_list before queueing */
1256                 mali_pp_job_list_remove(job);
1257
1258                 if (mali_scheduler_queue_pp_job(job)) {
1259                         /* Job queued successfully */
1260                         schedule_mask |= MALI_SCHEDULER_MASK_PP;
1261                 } else {
1262                         /* Failed to enqueue job, release job (with error) */
1263                         mali_pp_job_fb_lookup_remove(job);
1264                         mali_pp_job_mark_unstarted_failed(job);
1265
1266                         /* unlock scheduler in this uncommon case */
1267                         mali_scheduler_unlock();
1268
1269                         mali_timeline_tracker_release(
1270                                 mali_pp_job_get_tracker(job));
1271
1272                         /* Notify user space and close the job object */
1273                         mali_scheduler_complete_pp_job(job, 0, MALI_TRUE,
1274                                                        MALI_FALSE);
1275
1276                         mali_scheduler_lock();
1277                 }
1278         }
1279
1280         mali_scheduler_unlock();
1281
1282         /* Trigger scheduling of jobs */
1283         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1284 }
1285
1286 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */