drivers/gpu/arm/mali400/mali/common/mali_timeline.c

   1 /*
   2  * Copyright (C) 2013-2014 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_timeline.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_scheduler.h"
  14 #include "mali_soft_job.h"
  15 #include "mali_timeline_fence_wait.h"
  16 #include "mali_timeline_sync_fence.h"
  17 #include "mali_executor.h"
  18 #include "mali_pp_job.h"
  19
  20 #define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
  21
  22 /*
  23  * Following three elements are used to record how many
  24  * gp, physical pp or virtual pp jobs are delayed in the whole
  25  * timeline system, we can use these three value to decide
  26  * if need to deactivate idle group.
  27  */
  28 _mali_osk_atomic_t gp_tracker_count;
  29 _mali_osk_atomic_t phy_pp_tracker_count;
  30 _mali_osk_atomic_t virt_pp_tracker_count;
  31
  32 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
  33                 struct mali_timeline_waiter *waiter);
  34
  35 #if defined(CONFIG_SYNC)
  36 #include <linux/version.h>
  37 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
  38 #include <linux/list.h>
  39 #include <linux/workqueue.h>
  40 #include <linux/spinlock.h>
  41
  42 struct mali_deferred_fence_put_entry {
  43         struct hlist_node list;
  44         struct sync_fence *fence;
  45 };
  46
  47 static HLIST_HEAD(mali_timeline_sync_fence_to_free_list);
  48 static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock);
  49
  50 static void put_sync_fences(struct work_struct *ignore)
  51 {
  52         struct hlist_head list;
  53         struct hlist_node *tmp, *pos;
  54         unsigned long flags;
  55         struct mali_deferred_fence_put_entry *o;
  56
  57         spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
  58         hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list);
  59         spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
  60
  61         hlist_for_each_entry_safe(o, pos, tmp, &list, list) {
  62                 sync_fence_put(o->fence);
  63                 kfree(o);
  64         }
  65 }
  66
  67 static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences);
  68 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
  69
  70 /* Callback that is called when a sync fence a tracker is waiting on is signaled. */
  71 static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
  72 {
  73         struct mali_timeline_system  *system;
  74         struct mali_timeline_waiter  *waiter;
  75         struct mali_timeline_tracker *tracker;
  76         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  77         u32 tid = _mali_osk_get_tid();
  78         mali_bool is_aborting = MALI_FALSE;
  79 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
  80         int fence_status = sync_fence->status;
  81 #else
  82         int fence_status = atomic_read(&sync_fence->status);
  83 #endif
  84
  85         MALI_DEBUG_ASSERT_POINTER(sync_fence);
  86         MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
  87
  88         tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
  89         MALI_DEBUG_ASSERT_POINTER(tracker);
  90
  91         system = tracker->system;
  92         MALI_DEBUG_ASSERT_POINTER(system);
  93         MALI_DEBUG_ASSERT_POINTER(system->session);
  94
  95         mali_spinlock_reentrant_wait(system->spinlock, tid);
  96
  97         is_aborting = system->session->is_aborting;
  98         if (!is_aborting && (0 > fence_status)) {
  99                 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
 100                 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
 101         }
 102
 103         waiter = tracker->waiter_sync;
 104         MALI_DEBUG_ASSERT_POINTER(waiter);
 105
 106         tracker->sync_fence = NULL;
 107         tracker->fence.sync_fd = -1;
 108
 109         schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
 110
 111         /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */
 112         if (is_aborting) {
 113                 _mali_osk_wait_queue_wake_up(system->wait_queue);
 114         }
 115
 116         mali_spinlock_reentrant_signal(system->spinlock, tid);
 117
 118 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
 119         {
 120                 struct mali_deferred_fence_put_entry *obj;
 121
 122                 obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC);
 123                 if (obj) {
 124                         unsigned long flags;
 125                         mali_bool schedule = MALI_FALSE;
 126
 127                         obj->fence = sync_fence;
 128
 129                         spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
 130                         if (hlist_empty(&mali_timeline_sync_fence_to_free_list))
 131                                 schedule = MALI_TRUE;
 132                         hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list);
 133                         spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
 134
 135                         if (schedule)
 136                                 schedule_delayed_work(&delayed_sync_fence_put, 0);
 137                 }
 138         }
 139 #else
 140         sync_fence_put(sync_fence);
 141 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
 142
 143         if (!is_aborting) {
 144                 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
 145         }
 146 }
 147 #endif /* defined(CONFIG_SYNC) */
 148
 149 static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
 150 {
 151         MALI_DEBUG_ASSERT_POINTER(tracker);
 152         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
 153
 154         return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
 155 }
 156
 157 static void mali_timeline_timer_callback(void *data)
 158 {
 159         struct mali_timeline_system *system;
 160         struct mali_timeline_tracker *tracker;
 161         struct mali_timeline *timeline;
 162         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 163         u32 tid = _mali_osk_get_tid();
 164
 165         timeline = (struct mali_timeline *) data;
 166         MALI_DEBUG_ASSERT_POINTER(timeline);
 167
 168         system = timeline->system;
 169         MALI_DEBUG_ASSERT_POINTER(system);
 170
 171         mali_spinlock_reentrant_wait(system->spinlock, tid);
 172
 173         if (!system->timer_enabled) {
 174                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 175                 return;
 176         }
 177
 178         tracker = timeline->tracker_tail;
 179         timeline->timer_active = MALI_FALSE;
 180
 181         if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
 182                 /* This is likely the delayed work that has been schedule out before cancelled. */
 183                 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
 184                         mali_spinlock_reentrant_signal(system->spinlock, tid);
 185                         return;
 186                 }
 187
 188                 schedule_mask = mali_timeline_tracker_time_out(tracker);
 189                 tracker->timer_active = MALI_FALSE;
 190         } else {
 191                 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
 192         }
 193
 194         mali_spinlock_reentrant_signal(system->spinlock, tid);
 195
 196         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
 197 }
 198
 199 void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
 200 {
 201         u32 i;
 202         u32 tid = _mali_osk_get_tid();
 203
 204         MALI_DEBUG_ASSERT_POINTER(system);
 205
 206         mali_spinlock_reentrant_wait(system->spinlock, tid);
 207         system->timer_enabled = MALI_FALSE;
 208         mali_spinlock_reentrant_signal(system->spinlock, tid);
 209
 210         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 211                 struct mali_timeline *timeline = system->timelines[i];
 212
 213                 MALI_DEBUG_ASSERT_POINTER(timeline);
 214
 215                 if (NULL != timeline->delayed_work) {
 216                         _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
 217                         timeline->timer_active = MALI_FALSE;
 218                 }
 219         }
 220 }
 221
 222 static void mali_timeline_destroy(struct mali_timeline *timeline)
 223 {
 224         MALI_DEBUG_ASSERT_POINTER(timeline);
 225         if (NULL != timeline) {
 226                 /* Assert that the timeline object has been properly cleaned up before destroying it. */
 227                 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
 228                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
 229                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 230                 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
 231                 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
 232                 MALI_DEBUG_ASSERT(NULL != timeline->system);
 233                 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
 234
 235 #if defined(CONFIG_SYNC)
 236                 if (NULL != timeline->sync_tl) {
 237                         sync_timeline_destroy(timeline->sync_tl);
 238                 }
 239 #endif /* defined(CONFIG_SYNC) */
 240
 241                 if (NULL != timeline->delayed_work) {
 242                         _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
 243                         _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
 244                 }
 245
 246                 _mali_osk_free(timeline);
 247         }
 248 }
 249
 250 static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
 251 {
 252         struct mali_timeline *timeline;
 253
 254         MALI_DEBUG_ASSERT_POINTER(system);
 255         MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
 256
 257         timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
 258         if (NULL == timeline) {
 259                 return NULL;
 260         }
 261
 262         /* Initially the timeline is empty. */
 263 #if defined(MALI_TIMELINE_DEBUG_START_POINT)
 264         /* Start the timeline a bit before wrapping when debugging. */
 265         timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
 266 #else
 267         timeline->point_next = 1;
 268 #endif
 269         timeline->point_oldest = timeline->point_next;
 270
 271         /* The tracker and waiter lists will initially be empty. */
 272
 273         timeline->system = system;
 274         timeline->id = id;
 275
 276         timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
 277         if (NULL == timeline->delayed_work) {
 278                 mali_timeline_destroy(timeline);
 279                 return NULL;
 280         }
 281
 282         timeline->timer_active = MALI_FALSE;
 283
 284 #if defined(CONFIG_SYNC)
 285         {
 286                 char timeline_name[32];
 287
 288                 switch (id) {
 289                 case MALI_TIMELINE_GP:
 290                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
 291                         break;
 292                 case MALI_TIMELINE_PP:
 293                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
 294                         break;
 295                 case MALI_TIMELINE_SOFT:
 296                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
 297                         break;
 298                 default:
 299                         MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
 300                         mali_timeline_destroy(timeline);
 301                         return NULL;
 302                 }
 303
 304                 timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name);
 305                 if (NULL == timeline->sync_tl) {
 306                         mali_timeline_destroy(timeline);
 307                         return NULL;
 308                 }
 309         }
 310 #endif /* defined(CONFIG_SYNC) */
 311
 312         return timeline;
 313 }
 314
 315 static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
 316 {
 317         MALI_DEBUG_ASSERT_POINTER(timeline);
 318         MALI_DEBUG_ASSERT_POINTER(tracker);
 319
 320         if (mali_timeline_is_full(timeline)) {
 321                 /* Don't add tracker if timeline is full. */
 322                 tracker->point = MALI_TIMELINE_NO_POINT;
 323                 return;
 324         }
 325
 326         tracker->timeline = timeline;
 327         tracker->point    = timeline->point_next;
 328
 329         /* Find next available point. */
 330         timeline->point_next++;
 331         if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
 332                 timeline->point_next++;
 333         }
 334
 335         MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
 336
 337         if (MALI_TIMELINE_TRACKER_GP == tracker->type) {
 338                 _mali_osk_atomic_inc(&gp_tracker_count);
 339         } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) {
 340                 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
 341                         _mali_osk_atomic_inc(&virt_pp_tracker_count);
 342                 } else {
 343                         _mali_osk_atomic_inc(&phy_pp_tracker_count);
 344                 }
 345         }
 346
 347         /* Add tracker as new head on timeline's tracker list. */
 348         if (NULL == timeline->tracker_head) {
 349                 /* Tracker list is empty. */
 350                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 351
 352                 timeline->tracker_tail = tracker;
 353
 354                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
 355                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
 356         } else {
 357                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
 358
 359                 tracker->timeline_prev = timeline->tracker_head;
 360                 timeline->tracker_head->timeline_next = tracker;
 361
 362                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
 363         }
 364         timeline->tracker_head = tracker;
 365
 366         MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
 367         MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
 368 }
 369
 370 /* Inserting the waiter object into the given timeline */
 371 static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
 372 {
 373         struct mali_timeline_waiter *waiter_prev;
 374         struct mali_timeline_waiter *waiter_next;
 375
 376         /* Waiter time must be between timeline head and tail, and there must
 377          * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */
 378         MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
 379         MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
 380
 381         /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/
 382         waiter_prev = timeline->waiter_head; /* Insert new after  waiter_prev */
 383         waiter_next = NULL;                  /* Insert new before waiter_next */
 384
 385         /* Iterating backwards from head (newest) to tail (oldest) until we
 386          * find the correct spot to insert the new waiter */
 387         while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
 388                 waiter_next = waiter_prev;
 389                 waiter_prev = waiter_prev->timeline_prev;
 390         }
 391
 392         if (NULL == waiter_prev && NULL == waiter_next) {
 393                 /* list is empty */
 394                 timeline->waiter_head = waiter_new;
 395                 timeline->waiter_tail = waiter_new;
 396         } else if (NULL == waiter_next) {
 397                 /* insert at head */
 398                 waiter_new->timeline_prev = timeline->waiter_head;
 399                 timeline->waiter_head->timeline_next = waiter_new;
 400                 timeline->waiter_head = waiter_new;
 401         } else if (NULL == waiter_prev) {
 402                 /* insert at tail */
 403                 waiter_new->timeline_next = timeline->waiter_tail;
 404                 timeline->waiter_tail->timeline_prev = waiter_new;
 405                 timeline->waiter_tail = waiter_new;
 406         } else {
 407                 /* insert between */
 408                 waiter_new->timeline_next = waiter_next;
 409                 waiter_new->timeline_prev = waiter_prev;
 410                 waiter_next->timeline_prev = waiter_new;
 411                 waiter_prev->timeline_next = waiter_new;
 412         }
 413 }
 414
 415 static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
 416 {
 417         struct mali_timeline_system *system;
 418         struct mali_timeline_tracker *oldest_tracker;
 419
 420         MALI_DEBUG_ASSERT_POINTER(timeline);
 421         MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
 422
 423         system = timeline->system;
 424         MALI_DEBUG_ASSERT_POINTER(system);
 425
 426         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 427
 428         /* Timer is disabled, early out. */
 429         if (!system->timer_enabled) return;
 430
 431         oldest_tracker = timeline->tracker_tail;
 432         if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
 433                 if (MALI_FALSE == oldest_tracker->timer_active) {
 434                         if (MALI_TRUE == timeline->timer_active) {
 435                                 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
 436                         }
 437                         _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
 438                         oldest_tracker->timer_active = MALI_TRUE;
 439                         timeline->timer_active = MALI_TRUE;
 440                 }
 441         } else if (MALI_TRUE == timeline->timer_active) {
 442                 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
 443                 timeline->timer_active = MALI_FALSE;
 444         }
 445 }
 446
 447 static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
 448 {
 449         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 450
 451         MALI_DEBUG_ASSERT_POINTER(timeline);
 452
 453         MALI_DEBUG_CODE({
 454                 struct mali_timeline_system *system = timeline->system;
 455                 MALI_DEBUG_ASSERT_POINTER(system);
 456
 457                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 458         });
 459
 460         if (NULL != timeline->tracker_tail) {
 461                 /* Set oldest point to oldest tracker's point */
 462                 timeline->point_oldest = timeline->tracker_tail->point;
 463         } else {
 464                 /* No trackers, mark point list as empty */
 465                 timeline->point_oldest = timeline->point_next;
 466         }
 467
 468         /* Release all waiters no longer on the timeline's point list.
 469          * Releasing a waiter can trigger this function to be called again, so
 470          * we do not store any pointers on stack. */
 471         while (NULL != timeline->waiter_tail) {
 472                 u32 waiter_time_relative;
 473                 u32 time_head_relative;
 474                 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
 475
 476                 time_head_relative = timeline->point_next - timeline->point_oldest;
 477                 waiter_time_relative = waiter->point - timeline->point_oldest;
 478
 479                 if (waiter_time_relative < time_head_relative) {
 480                         /* This and all following waiters are on the point list, so we are done. */
 481                         break;
 482                 }
 483
 484                 /* Remove waiter from timeline's waiter list. */
 485                 if (NULL != waiter->timeline_next) {
 486                         waiter->timeline_next->timeline_prev = NULL;
 487                 } else {
 488                         /* This was the last waiter */
 489                         timeline->waiter_head = NULL;
 490                 }
 491                 timeline->waiter_tail = waiter->timeline_next;
 492
 493                 /* Release waiter.  This could activate a tracker, if this was
 494                  * the last waiter for the tracker. */
 495                 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
 496         }
 497
 498         return schedule_mask;
 499 }
 500
 501 void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
 502                                 mali_timeline_tracker_type type,
 503                                 struct mali_timeline_fence *fence,
 504                                 void *job)
 505 {
 506         MALI_DEBUG_ASSERT_POINTER(tracker);
 507         MALI_DEBUG_ASSERT_POINTER(job);
 508
 509         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
 510
 511         /* Zero out all tracker members. */
 512         _mali_osk_memset(tracker, 0, sizeof(*tracker));
 513
 514         tracker->type = type;
 515         tracker->job = job;
 516         tracker->trigger_ref_count = 1;  /* Prevents any callback from trigging while adding it */
 517         tracker->os_tick_create = _mali_osk_time_tickcount();
 518         MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
 519
 520         tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
 521
 522         /* Copy fence. */
 523         if (NULL != fence) {
 524                 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
 525         }
 526 }
 527
 528 mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
 529 {
 530         struct mali_timeline *timeline;
 531         struct mali_timeline_system *system;
 532         struct mali_timeline_tracker *tracker_next, *tracker_prev;
 533         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 534         u32 tid = _mali_osk_get_tid();
 535
 536         /* Upon entry a group lock will be held, but not a scheduler lock. */
 537         MALI_DEBUG_ASSERT_POINTER(tracker);
 538         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
 539
 540         /* Tracker should have been triggered */
 541         MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
 542
 543         /* All waiters should have been released at this point */
 544         MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
 545         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
 546
 547         MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
 548
 549         timeline = tracker->timeline;
 550         if (NULL == timeline) {
 551                 /* Tracker was not on a timeline, there is nothing to release. */
 552                 return MALI_SCHEDULER_MASK_EMPTY;
 553         }
 554
 555         system = timeline->system;
 556         MALI_DEBUG_ASSERT_POINTER(system);
 557
 558         mali_spinlock_reentrant_wait(system->spinlock, tid);
 559
 560         /* Tracker should still be on timeline */
 561         MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
 562         MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point));
 563
 564         /* Tracker is no longer valid. */
 565         MALI_DEBUG_CODE(tracker->magic = 0);
 566
 567         tracker_next = tracker->timeline_next;
 568         tracker_prev = tracker->timeline_prev;
 569         tracker->timeline_next = NULL;
 570         tracker->timeline_prev = NULL;
 571
 572         /* Removing tracker from timeline's tracker list */
 573         if (NULL == tracker_next) {
 574                 /* This tracker was the head */
 575                 timeline->tracker_head = tracker_prev;
 576         } else {
 577                 tracker_next->timeline_prev = tracker_prev;
 578         }
 579
 580         if (NULL == tracker_prev) {
 581                 /* This tracker was the tail */
 582                 timeline->tracker_tail = tracker_next;
 583                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 584                 /* Update the timeline's oldest time and release any waiters */
 585                 schedule_mask |= mali_timeline_update_oldest_point(timeline);
 586                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 587         } else {
 588                 tracker_prev->timeline_next = tracker_next;
 589         }
 590
 591         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 592
 593         /* Update delayed work only when it is the soft job timeline */
 594         if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
 595                 mali_timeline_update_delayed_work(tracker->timeline);
 596         }
 597
 598         mali_spinlock_reentrant_signal(system->spinlock, tid);
 599
 600         return schedule_mask;
 601 }
 602
 603 void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
 604                 struct mali_timeline_waiter *tail,
 605                 struct mali_timeline_waiter *head)
 606 {
 607         MALI_DEBUG_ASSERT_POINTER(system);
 608         MALI_DEBUG_ASSERT_POINTER(head);
 609         MALI_DEBUG_ASSERT_POINTER(tail);
 610         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 611
 612         head->tracker_next = system->waiter_empty_list;
 613         system->waiter_empty_list = tail;
 614 }
 615
 616 static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
 617 {
 618         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 619         struct mali_timeline_system *system;
 620         struct mali_timeline *timeline;
 621         u32 tid = _mali_osk_get_tid();
 622
 623         MALI_DEBUG_ASSERT_POINTER(tracker);
 624         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
 625
 626         system = tracker->system;
 627         MALI_DEBUG_ASSERT_POINTER(system);
 628         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 629
 630         tracker->os_tick_activate = _mali_osk_time_tickcount();
 631
 632         if (NULL != tracker->waiter_head) {
 633                 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
 634                 tracker->waiter_head = NULL;
 635                 tracker->waiter_tail = NULL;
 636         }
 637
 638         switch (tracker->type) {
 639         case MALI_TIMELINE_TRACKER_GP:
 640                 schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job);
 641
 642                 _mali_osk_atomic_dec(&gp_tracker_count);
 643                 break;
 644         case MALI_TIMELINE_TRACKER_PP:
 645                 schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job);
 646
 647                 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
 648                         _mali_osk_atomic_dec(&virt_pp_tracker_count);
 649                 } else {
 650                         _mali_osk_atomic_dec(&phy_pp_tracker_count);
 651                 }
 652                 break;
 653         case MALI_TIMELINE_TRACKER_SOFT:
 654                 timeline = tracker->timeline;
 655                 MALI_DEBUG_ASSERT_POINTER(timeline);
 656
 657                 mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
 658
 659                 /* Start a soft timer to make sure the soft job be released in a limited time */
 660                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 661                 mali_timeline_update_delayed_work(timeline);
 662                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 663                 break;
 664         case MALI_TIMELINE_TRACKER_WAIT:
 665                 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
 666                 break;
 667         case MALI_TIMELINE_TRACKER_SYNC:
 668 #if defined(CONFIG_SYNC)
 669                 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
 670 #else
 671                 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
 672 #endif /* defined(CONFIG_SYNC) */
 673                 break;
 674         default:
 675                 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
 676                 break;
 677         }
 678
 679         return schedule_mask;
 680 }
 681
 682 void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
 683 {
 684         u32 tid = _mali_osk_get_tid();
 685
 686         MALI_DEBUG_ASSERT_POINTER(tracker);
 687         MALI_DEBUG_ASSERT_POINTER(system);
 688
 689         mali_spinlock_reentrant_wait(system->spinlock, tid);
 690
 691         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
 692         tracker->trigger_ref_count++;
 693
 694         mali_spinlock_reentrant_signal(system->spinlock, tid);
 695 }
 696
 697 mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
 698 {
 699         u32 tid = _mali_osk_get_tid();
 700         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 701
 702         MALI_DEBUG_ASSERT_POINTER(tracker);
 703         MALI_DEBUG_ASSERT_POINTER(system);
 704
 705         mali_spinlock_reentrant_wait(system->spinlock, tid);
 706
 707         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
 708         tracker->trigger_ref_count--;
 709
 710         tracker->activation_error |= activation_error;
 711
 712         if (0 == tracker->trigger_ref_count) {
 713                 schedule_mask |= mali_timeline_tracker_activate(tracker);
 714                 tracker = NULL;
 715         }
 716
 717         mali_spinlock_reentrant_signal(system->spinlock, tid);
 718
 719         return schedule_mask;
 720 }
 721
 722 void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
 723 {
 724         u32 i;
 725
 726         MALI_DEBUG_ASSERT_POINTER(fence);
 727         MALI_DEBUG_ASSERT_POINTER(uk_fence);
 728
 729         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 730                 fence->points[i] = uk_fence->points[i];
 731         }
 732
 733         fence->sync_fd = uk_fence->sync_fd;
 734 }
 735
 736 struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
 737 {
 738         u32 i;
 739         struct mali_timeline_system *system;
 740
 741         MALI_DEBUG_ASSERT_POINTER(session);
 742         MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
 743
 744         system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
 745         if (NULL == system) {
 746                 return NULL;
 747         }
 748
 749         system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
 750         if (NULL == system->spinlock) {
 751                 mali_timeline_system_destroy(system);
 752                 return NULL;
 753         }
 754
 755         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 756                 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
 757                 if (NULL == system->timelines[i]) {
 758                         mali_timeline_system_destroy(system);
 759                         return NULL;
 760                 }
 761         }
 762
 763 #if defined(CONFIG_SYNC)
 764         system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled");
 765         if (NULL == system->signaled_sync_tl) {
 766                 mali_timeline_system_destroy(system);
 767                 return NULL;
 768         }
 769 #endif /* defined(CONFIG_SYNC) */
 770
 771         system->waiter_empty_list = NULL;
 772         system->session = session;
 773         system->timer_enabled = MALI_TRUE;
 774
 775         system->wait_queue = _mali_osk_wait_queue_init();
 776         if (NULL == system->wait_queue) {
 777                 mali_timeline_system_destroy(system);
 778                 return NULL;
 779         }
 780
 781         return system;
 782 }
 783
 784 #if defined(CONFIG_SYNC)
 785
 786 /**
 787  * Check if there are any trackers left on timeline.
 788  *
 789  * Used as a wait queue conditional.
 790  *
 791  * @param data Timeline.
 792  * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not.
 793  */
 794 static mali_bool mali_timeline_has_no_trackers(void *data)
 795 {
 796         struct mali_timeline *timeline = (struct mali_timeline *) data;
 797
 798         MALI_DEBUG_ASSERT_POINTER(timeline);
 799
 800         return mali_timeline_is_empty(timeline);
 801 }
 802
 803 /**
 804  * Cancel sync fence waiters waited upon by trackers on all timelines.
 805  *
 806  * Will return after all timelines have no trackers left.
 807  *
 808  * @param system Timeline system.
 809  */
 810 static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
 811 {
 812         u32 i;
 813         u32 tid = _mali_osk_get_tid();
 814         struct mali_timeline_tracker *tracker, *tracker_next;
 815         _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
 816
 817         MALI_DEBUG_ASSERT_POINTER(system);
 818         MALI_DEBUG_ASSERT_POINTER(system->session);
 819         MALI_DEBUG_ASSERT(system->session->is_aborting);
 820
 821         mali_spinlock_reentrant_wait(system->spinlock, tid);
 822
 823         /* Cancel sync fence waiters. */
 824         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 825                 struct mali_timeline *timeline = system->timelines[i];
 826
 827                 MALI_DEBUG_ASSERT_POINTER(timeline);
 828
 829                 tracker_next = timeline->tracker_tail;
 830                 while (NULL != tracker_next) {
 831                         tracker = tracker_next;
 832                         tracker_next = tracker->timeline_next;
 833
 834                         if (NULL == tracker->sync_fence) continue;
 835
 836                         MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
 837
 838                         /* Cancel sync fence waiter. */
 839                         if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
 840                                 /* Callback was not called, move tracker to local list. */
 841                                 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
 842                         }
 843                 }
 844         }
 845
 846         mali_spinlock_reentrant_signal(system->spinlock, tid);
 847
 848         /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */
 849         _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
 850                 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
 851         }
 852
 853         /* Sleep until all sync fence callbacks are done and all timelines are empty. */
 854         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 855                 struct mali_timeline *timeline = system->timelines[i];
 856
 857                 MALI_DEBUG_ASSERT_POINTER(timeline);
 858
 859                 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
 860         }
 861 }
 862
 863 #endif /* defined(CONFIG_SYNC) */
 864
 865 void mali_timeline_system_abort(struct mali_timeline_system *system)
 866 {
 867         MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
 868
 869         MALI_DEBUG_ASSERT_POINTER(system);
 870         MALI_DEBUG_ASSERT_POINTER(system->session);
 871         MALI_DEBUG_ASSERT(system->session->is_aborting);
 872
 873         MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
 874
 875 #if defined(CONFIG_SYNC)
 876         mali_timeline_cancel_sync_fence_waiters(system);
 877 #endif /* defined(CONFIG_SYNC) */
 878
 879         /* Should not be any waiters or trackers left at this point. */
 880         MALI_DEBUG_CODE({
 881                 u32 i;
 882                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 883                 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
 884                 {
 885                         struct mali_timeline *timeline = system->timelines[i];
 886                         MALI_DEBUG_ASSERT_POINTER(timeline);
 887                         MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
 888                         MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
 889                         MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 890                         MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
 891                         MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
 892                 }
 893                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 894         });
 895 }
 896
 897 void mali_timeline_system_destroy(struct mali_timeline_system *system)
 898 {
 899         u32 i;
 900         struct mali_timeline_waiter *waiter, *next;
 901
 902         MALI_DEBUG_ASSERT_POINTER(system);
 903         MALI_DEBUG_ASSERT_POINTER(system->session);
 904
 905         MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
 906
 907         if (NULL != system) {
 908                 /* There should be no waiters left on this queue. */
 909                 if (NULL != system->wait_queue) {
 910                         _mali_osk_wait_queue_term(system->wait_queue);
 911                         system->wait_queue = NULL;
 912                 }
 913
 914                 /* Free all waiters in empty list */
 915                 waiter = system->waiter_empty_list;
 916                 while (NULL != waiter) {
 917                         next = waiter->tracker_next;
 918                         _mali_osk_free(waiter);
 919                         waiter = next;
 920                 }
 921
 922 #if defined(CONFIG_SYNC)
 923                 if (NULL != system->signaled_sync_tl) {
 924                         sync_timeline_destroy(system->signaled_sync_tl);
 925                 }
 926 #endif /* defined(CONFIG_SYNC) */
 927
 928                 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 929                         if (NULL != system->timelines[i]) {
 930                                 mali_timeline_destroy(system->timelines[i]);
 931                         }
 932                 }
 933                 if (NULL != system->spinlock) {
 934                         mali_spinlock_reentrant_term(system->spinlock);
 935                 }
 936
 937                 _mali_osk_free(system);
 938         }
 939 }
 940
 941 /**
 942  * Find how many waiters are needed for a given fence.
 943  *
 944  * @param fence The fence to check.
 945  * @return Number of waiters needed for fence.
 946  */
 947 static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
 948 {
 949         u32 i, num_waiters = 0;
 950
 951         MALI_DEBUG_ASSERT_POINTER(fence);
 952
 953         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 954                 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
 955                         ++num_waiters;
 956                 }
 957         }
 958
 959 #if defined(CONFIG_SYNC)
 960         if (-1 != fence->sync_fd) ++num_waiters;
 961 #endif /* defined(CONFIG_SYNC) */
 962
 963         return num_waiters;
 964 }
 965
 966 static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
 967 {
 968         struct mali_timeline_waiter *waiter;
 969
 970         MALI_DEBUG_ASSERT_POINTER(system);
 971         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 972
 973         waiter = system->waiter_empty_list;
 974         if (NULL != waiter) {
 975                 /* Remove waiter from empty list and zero it */
 976                 system->waiter_empty_list = waiter->tracker_next;
 977                 _mali_osk_memset(waiter, 0, sizeof(*waiter));
 978         }
 979
 980         /* Return NULL if list was empty. */
 981         return waiter;
 982 }
 983
 984 static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
 985                 struct mali_timeline_waiter **tail,
 986                 struct mali_timeline_waiter **head,
 987                 int max_num_waiters)
 988 {
 989         u32 i, tid = _mali_osk_get_tid();
 990         mali_bool do_alloc;
 991         struct mali_timeline_waiter *waiter;
 992
 993         MALI_DEBUG_ASSERT_POINTER(system);
 994         MALI_DEBUG_ASSERT_POINTER(tail);
 995         MALI_DEBUG_ASSERT_POINTER(head);
 996
 997         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 998
 999         *head = *tail = NULL;
1000         do_alloc = MALI_FALSE;
1001         i = 0;
1002         while (i < max_num_waiters) {
1003                 if (MALI_FALSE == do_alloc) {
1004                         waiter = mali_timeline_system_get_zeroed_waiter(system);
1005                         if (NULL == waiter) {
1006                                 do_alloc = MALI_TRUE;
1007                                 mali_spinlock_reentrant_signal(system->spinlock, tid);
1008                                 continue;
1009                         }
1010                 } else {
1011                         waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
1012                         if (NULL == waiter) break;
1013                 }
1014                 ++i;
1015                 if (NULL == *tail) {
1016                         *tail = waiter;
1017                         *head = waiter;
1018                 } else {
1019                         (*head)->tracker_next = waiter;
1020                         *head = waiter;
1021                 }
1022         }
1023         if (MALI_TRUE == do_alloc) {
1024                 mali_spinlock_reentrant_wait(system->spinlock, tid);
1025         }
1026 }
1027
1028 /**
1029  * Create waiters for the given tracker. The tracker is activated when all waiters are release.
1030  *
1031  * @note Tracker can potentially be activated before this function returns.
1032  *
1033  * @param system Timeline system.
1034  * @param tracker Tracker we will create waiters for.
1035  * @param waiter_tail List of pre-allocated waiters.
1036  * @param waiter_head List of pre-allocated waiters.
1037  */
1038 static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
1039                 struct mali_timeline_tracker *tracker,
1040                 struct mali_timeline_waiter *waiter_tail,
1041                 struct mali_timeline_waiter *waiter_head)
1042 {
1043         int i;
1044         u32 tid = _mali_osk_get_tid();
1045         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1046 #if defined(CONFIG_SYNC)
1047         struct sync_fence *sync_fence = NULL;
1048 #endif /* defined(CONFIG_SYNC) */
1049
1050         MALI_DEBUG_ASSERT_POINTER(system);
1051         MALI_DEBUG_ASSERT_POINTER(tracker);
1052
1053         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1054
1055         MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
1056         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1057         MALI_DEBUG_ASSERT(NULL != tracker->job);
1058
1059         /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter
1060          * into the timelines sorted list of waiters */
1061         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1062                 mali_timeline_point point;
1063                 struct mali_timeline *timeline;
1064                 struct mali_timeline_waiter *waiter;
1065
1066                 /* Get point on current timeline from tracker's fence. */
1067                 point = tracker->fence.points[i];
1068
1069                 if (likely(MALI_TIMELINE_NO_POINT == point)) {
1070                         /* Fence contains no point on this timeline so we don't need a waiter. */
1071                         continue;
1072                 }
1073
1074                 timeline = system->timelines[i];
1075                 MALI_DEBUG_ASSERT_POINTER(timeline);
1076
1077                 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
1078                         MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
1079                                           point, timeline->point_oldest, timeline->point_next));
1080                         continue;
1081                 }
1082
1083                 if (likely(mali_timeline_is_point_released(timeline, point))) {
1084                         /* Tracker representing the point has been released so we don't need a
1085                          * waiter. */
1086                         continue;
1087                 }
1088
1089                 /* The point is on timeline. */
1090                 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
1091
1092                 /* Get a new zeroed waiter object. */
1093                 if (likely(NULL != waiter_tail)) {
1094                         waiter = waiter_tail;
1095                         waiter_tail = waiter_tail->tracker_next;
1096                 } else {
1097                         MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1098                         continue;
1099                 }
1100
1101                 /* Yanking the trigger ref count of the tracker. */
1102                 tracker->trigger_ref_count++;
1103
1104                 waiter->point   = point;
1105                 waiter->tracker = tracker;
1106
1107                 /* Insert waiter on tracker's singly-linked waiter list. */
1108                 if (NULL == tracker->waiter_head) {
1109                         /* list is empty */
1110                         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1111                         tracker->waiter_tail = waiter;
1112                 } else {
1113                         tracker->waiter_head->tracker_next = waiter;
1114                 }
1115                 tracker->waiter_head = waiter;
1116
1117                 /* Add waiter to timeline. */
1118                 mali_timeline_insert_waiter(timeline, waiter);
1119         }
1120 #if defined(CONFIG_SYNC)
1121         if (-1 != tracker->fence.sync_fd) {
1122                 int ret;
1123                 struct mali_timeline_waiter *waiter;
1124
1125                 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1126                 if (unlikely(NULL == sync_fence)) {
1127                         MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1128                         goto exit;
1129                 }
1130
1131                 /* Check if we have a zeroed waiter object available. */
1132                 if (unlikely(NULL == waiter_tail)) {
1133                         MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1134                         goto exit;
1135                 }
1136
1137                 /* Start asynchronous wait that will release waiter when the fence is signaled. */
1138                 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1139                 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1140                 if (1 == ret) {
1141                         /* Fence already signaled, no waiter needed. */
1142                         tracker->fence.sync_fd = -1;
1143                         goto exit;
1144                 } else if (0 != ret) {
1145                         MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1146                         tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1147                         goto exit;
1148                 }
1149
1150                 /* Grab new zeroed waiter object. */
1151                 waiter = waiter_tail;
1152                 waiter_tail = waiter_tail->tracker_next;
1153
1154                 /* Increase the trigger ref count of the tracker. */
1155                 tracker->trigger_ref_count++;
1156
1157                 waiter->point   = MALI_TIMELINE_NO_POINT;
1158                 waiter->tracker = tracker;
1159
1160                 /* Insert waiter on tracker's singly-linked waiter list. */
1161                 if (NULL == tracker->waiter_head) {
1162                         /* list is empty */
1163                         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1164                         tracker->waiter_tail = waiter;
1165                 } else {
1166                         tracker->waiter_head->tracker_next = waiter;
1167                 }
1168                 tracker->waiter_head = waiter;
1169
1170                 /* Also store waiter in separate field for easy access by sync callback. */
1171                 tracker->waiter_sync = waiter;
1172
1173                 /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */
1174                 tracker->sync_fence = sync_fence;
1175
1176                 sync_fence = NULL;
1177         }
1178 exit:
1179 #endif /* defined(CONFIG_SYNC) */
1180
1181         if (NULL != waiter_tail) {
1182                 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1183         }
1184
1185         /* Release the initial trigger ref count. */
1186         tracker->trigger_ref_count--;
1187
1188         /* If there were no waiters added to this tracker we activate immediately. */
1189         if (0 == tracker->trigger_ref_count) {
1190                 schedule_mask |= mali_timeline_tracker_activate(tracker);
1191         }
1192
1193         mali_spinlock_reentrant_signal(system->spinlock, tid);
1194
1195 #if defined(CONFIG_SYNC)
1196         if (NULL != sync_fence) {
1197                 sync_fence_put(sync_fence);
1198         }
1199 #endif /* defined(CONFIG_SYNC) */
1200
1201         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1202 }
1203
1204 mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
1205                 struct mali_timeline_tracker *tracker,
1206                 enum mali_timeline_id timeline_id)
1207 {
1208         int num_waiters = 0;
1209         struct mali_timeline_waiter *waiter_tail, *waiter_head;
1210         u32 tid = _mali_osk_get_tid();
1211         mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1212
1213         MALI_DEBUG_ASSERT_POINTER(system);
1214         MALI_DEBUG_ASSERT_POINTER(system->session);
1215         MALI_DEBUG_ASSERT_POINTER(tracker);
1216
1217         MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1218         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1219         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1220
1221         MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1222
1223         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1224         tracker->system = system;
1225
1226         mali_spinlock_reentrant_wait(system->spinlock, tid);
1227
1228         num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1229
1230         /* Allocate waiters. */
1231         mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1232         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1233
1234         /* Add tracker to timeline.  This will allocate a point for the tracker on the timeline. If
1235          * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the
1236          * point will be MALI_TIMELINE_NO_POINT.
1237          *
1238          * NOTE: the tracker can fail to be added if the timeline is full.  If this happens, the
1239          * point will be MALI_TIMELINE_NO_POINT. */
1240         MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1241         if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1242                 struct mali_timeline *timeline = system->timelines[timeline_id];
1243                 mali_timeline_insert_tracker(timeline, tracker);
1244                 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1245         }
1246
1247         point = tracker->point;
1248
1249         /* Create waiters for tracker based on supplied fence.  Each waiter will increase the
1250          * trigger ref count. */
1251         mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1252         tracker = NULL;
1253
1254         /* At this point the tracker object might have been freed so we should no longer
1255          * access it. */
1256
1257
1258         /* The tracker will always be activated after calling add_tracker, even if NO_POINT is
1259          * returned. */
1260         return point;
1261 }
1262
1263 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
1264                 struct mali_timeline_waiter *waiter)
1265 {
1266         struct mali_timeline_tracker *tracker;
1267         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1268
1269         MALI_DEBUG_ASSERT_POINTER(system);
1270         MALI_DEBUG_ASSERT_POINTER(waiter);
1271
1272         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1273
1274         tracker = waiter->tracker;
1275         MALI_DEBUG_ASSERT_POINTER(tracker);
1276
1277         /* At this point the waiter has been removed from the timeline's waiter list, but it is
1278          * still on the tracker's waiter list.  All of the tracker's waiters will be released when
1279          * the tracker is activated. */
1280
1281         waiter->point   = MALI_TIMELINE_NO_POINT;
1282         waiter->tracker = NULL;
1283
1284         tracker->trigger_ref_count--;
1285         if (0 == tracker->trigger_ref_count) {
1286                 /* This was the last waiter; activate tracker */
1287                 schedule_mask |= mali_timeline_tracker_activate(tracker);
1288                 tracker = NULL;
1289         }
1290
1291         return schedule_mask;
1292 }
1293
1294 mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
1295                 enum mali_timeline_id timeline_id)
1296 {
1297         mali_timeline_point point;
1298         struct mali_timeline *timeline;
1299         u32 tid = _mali_osk_get_tid();
1300
1301         MALI_DEBUG_ASSERT_POINTER(system);
1302
1303         if (MALI_TIMELINE_MAX <= timeline_id) {
1304                 return MALI_TIMELINE_NO_POINT;
1305         }
1306
1307         mali_spinlock_reentrant_wait(system->spinlock, tid);
1308
1309         timeline = system->timelines[timeline_id];
1310         MALI_DEBUG_ASSERT_POINTER(timeline);
1311
1312         point = MALI_TIMELINE_NO_POINT;
1313         if (timeline->point_oldest != timeline->point_next) {
1314                 point = timeline->point_next - 1;
1315                 if (MALI_TIMELINE_NO_POINT == point) point--;
1316         }
1317
1318         mali_spinlock_reentrant_signal(system->spinlock, tid);
1319
1320         return point;
1321 }
1322
1323 void mali_timeline_initialize(void)
1324 {
1325         _mali_osk_atomic_init(&gp_tracker_count, 0);
1326         _mali_osk_atomic_init(&phy_pp_tracker_count, 0);
1327         _mali_osk_atomic_init(&virt_pp_tracker_count, 0);
1328 }
1329
1330 void mali_timeline_terminate(void)
1331 {
1332         _mali_osk_atomic_term(&gp_tracker_count);
1333         _mali_osk_atomic_term(&phy_pp_tracker_count);
1334         _mali_osk_atomic_term(&virt_pp_tracker_count);
1335 }
1336
1337 #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1338
1339 static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1340 {
1341         struct mali_timeline *timeline;
1342         struct mali_timeline_system *system;
1343
1344         MALI_DEBUG_ASSERT_POINTER(tracker);
1345
1346         MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1347         timeline = tracker->timeline;
1348
1349         MALI_DEBUG_ASSERT_POINTER(timeline->system);
1350         system = timeline->system;
1351
1352         if (MALI_TIMELINE_MAX > id) {
1353                 if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) {
1354                         return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1355                 } else {
1356                         return MALI_FALSE;
1357                 }
1358         } else {
1359                 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1360                 return MALI_FALSE;
1361         }
1362 }
1363
1364 static const char *timeline_id_to_string(enum mali_timeline_id id)
1365 {
1366         switch (id) {
1367         case MALI_TIMELINE_GP:
1368                 return "GP";
1369         case MALI_TIMELINE_PP:
1370                 return "PP";
1371         case MALI_TIMELINE_SOFT:
1372                 return "SOFT";
1373         default:
1374                 return "NONE";
1375         }
1376 }
1377
1378 static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1379 {
1380         switch (type) {
1381         case MALI_TIMELINE_TRACKER_GP:
1382                 return "GP";
1383         case MALI_TIMELINE_TRACKER_PP:
1384                 return "PP";
1385         case MALI_TIMELINE_TRACKER_SOFT:
1386                 return "SOFT";
1387         case MALI_TIMELINE_TRACKER_WAIT:
1388                 return "WAIT";
1389         case MALI_TIMELINE_TRACKER_SYNC:
1390                 return "SYNC";
1391         default:
1392                 return "INVALID";
1393         }
1394 }
1395
1396 mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1397 {
1398         struct mali_timeline *timeline = NULL;
1399
1400         MALI_DEBUG_ASSERT_POINTER(tracker);
1401         timeline = tracker->timeline;
1402
1403         if (0 != tracker->trigger_ref_count) {
1404                 return MALI_TIMELINE_TS_WAITING;
1405         }
1406
1407         if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1408                 return MALI_TIMELINE_TS_ACTIVE;
1409         }
1410
1411         if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1412                 return MALI_TIMELINE_TS_INIT;
1413         }
1414
1415         return MALI_TIMELINE_TS_FINISH;
1416 }
1417
1418 void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx)
1419 {
1420         const char *tracker_state = "IWAF";
1421         char state_char = 'I';
1422         char tracker_type[32] = {0};
1423
1424         MALI_DEBUG_ASSERT_POINTER(tracker);
1425
1426         state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1427         _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1428
1429 #if defined(CONFIG_SYNC)
1430         if (0 != tracker->trigger_ref_count) {
1431                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)]  job:(0x%08X)\n",
1432                                     tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1433                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1434                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1435                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1436                                     tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1437         } else {
1438                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c  fd:%d  fence:(0x%08X)  job:(0x%08X)\n",
1439                                     tracker_type, tracker->point, state_char,
1440                                     tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1441         }
1442 #else
1443         if (0 != tracker->trigger_ref_count) {
1444                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)]  job:(0x%08X)\n",
1445                                     tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1446                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1447                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1448                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1449                                     tracker->job);
1450         } else {
1451                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c  job:(0x%08X)\n",
1452                                     tracker_type, tracker->point, state_char,
1453                                     tracker->job);
1454         }
1455 #endif
1456 }
1457
1458 void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx)
1459 {
1460         struct mali_timeline_tracker *tracker = NULL;
1461
1462         MALI_DEBUG_ASSERT_POINTER(timeline);
1463
1464         tracker = timeline->tracker_tail;
1465         while (NULL != tracker) {
1466                 mali_timeline_debug_print_tracker(tracker, print_ctx);
1467                 tracker = tracker->timeline_next;
1468         }
1469 }
1470
1471 void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx)
1472 {
1473         int i;
1474         int num_printed = 0;
1475         u32 tid = _mali_osk_get_tid();
1476
1477         MALI_DEBUG_ASSERT_POINTER(system);
1478
1479         mali_spinlock_reentrant_wait(system->spinlock, tid);
1480
1481         /* Print all timelines */
1482         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1483                 struct mali_timeline *timeline = system->timelines[i];
1484
1485                 MALI_DEBUG_ASSERT_POINTER(timeline);
1486
1487                 if (NULL == timeline->tracker_head) continue;
1488
1489                 _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n",
1490                                     timeline_id_to_string((enum mali_timeline_id)i));
1491
1492                 mali_timeline_debug_print_timeline(timeline, print_ctx);
1493                 num_printed++;
1494         }
1495
1496         if (0 == num_printed) {
1497                 _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n");
1498         }
1499
1500         mali_spinlock_reentrant_signal(system->spinlock, tid);
1501 }
1502
1503 #endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */