drivers/staging/imgtec/rogue/devicemem_history_server.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          Devicemem history functions
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Devicemem history functions
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43
  44 #include "allocmem.h"
  45 #include "pmr.h"
  46 #include "pvrsrv.h"
  47 #include "pvrsrv_device.h"
  48 #include "pvr_debug.h"
  49 #include "devicemem_server.h"
  50 #include "lock.h"
  51 #include "devicemem_history_server.h"
  52 #include "pdump_km.h"
  53
  54 #define ALLOCATION_LIST_NUM_ENTRIES 10000
  55
  56 /* data type to hold an allocation index.
  57  * we make it 16 bits wide if possible
  58  */
  59 #if ALLOCATION_LIST_NUM_ENTRIES <= 0xFFFF
  60 typedef uint16_t ALLOC_INDEX_T;
  61 #else
  62 typedef uint32_t ALLOC_INDEX_T;
  63 #endif
  64
  65 /* a record describing a single allocation known to DeviceMemHistory.
  66  * this is an element in a doubly linked list of allocations
  67  */
  68 typedef struct _RECORD_ALLOCATION_
  69 {
  70         /* time when this RECORD_ALLOCATION was created/initialised */
  71         IMG_UINT64 ui64CreationTime;
  72         /* serial number of the PMR relating to this allocation */
  73         IMG_UINT64 ui64Serial;
  74         /* base DevVAddr of this allocation */
  75         IMG_DEV_VIRTADDR sDevVAddr;
  76         /* size in bytes of this allocation */
  77         IMG_DEVMEM_SIZE_T uiSize;
  78         /* Log2 page size of this allocation's GPU pages */
  79         IMG_UINT32 ui32Log2PageSize;
  80         /* Process ID (PID) this allocation belongs to */
  81         IMG_PID uiPID;
  82         /* index of previous allocation in the list */
  83         ALLOC_INDEX_T ui32Prev;
  84         /* index of next allocation in the list */
  85         ALLOC_INDEX_T ui32Next;
  86         /* annotation/name of this allocation */
  87         IMG_CHAR szName[DEVICEMEM_HISTORY_TEXT_BUFSZ];
  88 } RECORD_ALLOCATION;
  89
  90 /* each command in the circular buffer is prefixed with an 8-bit value
  91  * denoting the command type
  92  */
  93 typedef enum _COMMAND_TYPE_
  94 {
  95         COMMAND_TYPE_NONE,
  96         COMMAND_TYPE_TIMESTAMP,
  97         COMMAND_TYPE_MAP_ALL,
  98         COMMAND_TYPE_UNMAP_ALL,
  99         COMMAND_TYPE_MAP_RANGE,
 100         COMMAND_TYPE_UNMAP_RANGE,
 101         /* sentinel value */
 102         COMMAND_TYPE_COUNT,
 103 } COMMAND_TYPE;
 104
 105 /* Timestamp command:
 106  * This command is inserted into the circular buffer to provide an updated
 107  * timestamp.
 108  * The nanosecond-accuracy timestamp is packed into a 56-bit integer, in order
 109  * for the whole command to fit into 8 bytes.
 110  */
 111 typedef struct _COMMAND_TIMESTAMP_
 112 {
 113         IMG_UINT8 aui8TimeNs[7];
 114 } COMMAND_TIMESTAMP;
 115
 116 /* MAP_ALL command:
 117  * This command denotes the allocation at the given index was wholly mapped
 118  * in to the GPU MMU
 119  */
 120 typedef struct _COMMAND_MAP_ALL_
 121 {
 122         ALLOC_INDEX_T uiAllocIndex;
 123 } COMMAND_MAP_ALL;
 124
 125 /* UNMAP_ALL command:
 126  * This command denotes the allocation at the given index was wholly unmapped
 127  * from the GPU MMU
 128  * Note: COMMAND_MAP_ALL and COMMAND_UNMAP_ALL commands have the same layout.
 129  */
 130 typedef COMMAND_MAP_ALL COMMAND_UNMAP_ALL;
 131
 132 /* packing attributes for the MAP_RANGE command */
 133 #define MAP_RANGE_MAX_START ((1 << 18) - 1)
 134 #define MAP_RANGE_MAX_RANGE ((1 << 12) - 1)
 135
 136 /* MAP_RANGE command:
 137  * Denotes a range of pages within the given allocation being mapped.
 138  * The range is expressed as [Page Index] + [Page Count]
 139  * This information is packed into a 40-bit integer, in order to make
 140  * the command size 8 bytes.
 141  */
 142
 143 typedef struct _COMMAND_MAP_RANGE_
 144 {
 145         IMG_UINT8 aui8Data[5];
 146         ALLOC_INDEX_T uiAllocIndex;
 147 } COMMAND_MAP_RANGE;
 148
 149 /* UNMAP_RANGE command:
 150  * Denotes a range of pages within the given allocation being mapped.
 151  * The range is expressed as [Page Index] + [Page Count]
 152  * This information is packed into a 40-bit integer, in order to make
 153  * the command size 8 bytes.
 154  * Note: COMMAND_MAP_RANGE and COMMAND_UNMAP_RANGE commands have the same layout.
 155  */
 156 typedef COMMAND_MAP_RANGE COMMAND_UNMAP_RANGE;
 157
 158 /* wrapper structure for a command */
 159 typedef struct _COMMAND_WRAPPER_
 160 {
 161         IMG_UINT8 ui8Type;
 162         union {
 163                 COMMAND_TIMESTAMP sTimeStamp;
 164                 COMMAND_MAP_ALL sMapAll;
 165                 COMMAND_UNMAP_ALL sUnmapAll;
 166                 COMMAND_MAP_RANGE sMapRange;
 167                 COMMAND_UNMAP_RANGE sUnmapRange;
 168         } u;
 169 } COMMAND_WRAPPER;
 170
 171 /* target size for the circular buffer of commands */
 172 #define CIRCULAR_BUFFER_SIZE_KB 2048
 173 /* turn the circular buffer target size into a number of commands */
 174 #define CIRCULAR_BUFFER_NUM_COMMANDS ((CIRCULAR_BUFFER_SIZE_KB * 1024) / sizeof(COMMAND_WRAPPER))
 175
 176 /* index value denoting the end of a list */
 177 #define END_OF_LIST 0xFFFFFFFF
 178 #define ALLOC_INDEX_TO_PTR(idx) (&(gsDevicememHistoryData.sRecords.pasAllocations[idx]))
 179 #define CHECK_ALLOC_INDEX(idx) (idx < ALLOCATION_LIST_NUM_ENTRIES)
 180
 181 /* wrapper structure for the allocation records and the commands circular buffer */
 182 typedef struct _RECORDS_
 183 {
 184         RECORD_ALLOCATION *pasAllocations;
 185         IMG_UINT32 ui32AllocationsListHead;
 186
 187         IMG_UINT32 ui32Head;
 188         IMG_UINT32 ui32Tail;
 189         COMMAND_WRAPPER *pasCircularBuffer;;
 190 } RECORDS;
 191
 192 typedef struct _DEVICEMEM_HISTORY_DATA_
 193 {
 194         /* debugfs entry */
 195         void *pvStatsEntry;
 196
 197         RECORDS sRecords;
 198         POS_LOCK hLock;
 199 } DEVICEMEM_HISTORY_DATA;
 200
 201 static DEVICEMEM_HISTORY_DATA gsDevicememHistoryData = { 0 };
 202
 203 static void DevicememHistoryLock(void)
 204 {
 205         OSLockAcquire(gsDevicememHistoryData.hLock);
 206 }
 207
 208 static void DevicememHistoryUnlock(void)
 209 {
 210         OSLockRelease(gsDevicememHistoryData.hLock);
 211 }
 212
 213 /* given a time stamp, calculate the age in nanoseconds */
 214 static IMG_UINT64 _CalculateAge(IMG_UINT64 ui64Now,
 215                                                 IMG_UINT64 ui64Then,
 216                                                 IMG_UINT64 ui64Max)
 217 {
 218         if(ui64Now >= ui64Then)
 219         {
 220                 /* no clock wrap */
 221                 return ui64Now - ui64Then;
 222         }
 223         else
 224         {
 225                 /* clock has wrapped */
 226                 return (ui64Max - ui64Then) + ui64Now + 1;
 227         }
 228 }
 229
 230 /* AcquireCBSlot:
 231  * Acquire the next slot in the circular buffer and
 232  * move the circular buffer head along by one
 233  * Returns a pointer to the acquired slot.
 234  */
 235 static COMMAND_WRAPPER *AcquireCBSlot(void)
 236 {
 237         COMMAND_WRAPPER *psSlot;
 238
 239         psSlot = &gsDevicememHistoryData.sRecords.pasCircularBuffer[gsDevicememHistoryData.sRecords.ui32Head];
 240
 241         gsDevicememHistoryData.sRecords.ui32Head =
 242                 (gsDevicememHistoryData.sRecords.ui32Head + 1)
 243                                 % CIRCULAR_BUFFER_NUM_COMMANDS;
 244
 245         return psSlot;
 246 }
 247
 248 /* TimeStampPack:
 249  * Packs the given timestamp value into the COMMAND_TIMESTAMP structure.
 250  * This takes a 64-bit nanosecond timestamp and packs it in to a 56-bit
 251  * integer in the COMMAND_TIMESTAMP command.
 252  */
 253 static void TimeStampPack(COMMAND_TIMESTAMP *psTimeStamp, IMG_UINT64 ui64Now)
 254 {
 255         IMG_UINT32 i;
 256
 257         for(i = 0; i < IMG_ARR_NUM_ELEMS(psTimeStamp->aui8TimeNs); i++)
 258         {
 259                 psTimeStamp->aui8TimeNs[i] = ui64Now & 0xFF;
 260                 ui64Now >>= 8;
 261         }
 262 }
 263
 264 /* packing a 64-bit nanosecond into a 7-byte integer loses the
 265  * top 8 bits of data. This must be taken into account when
 266  * comparing a full timestamp against an unpacked timestamp
 267  */
 268 #define TIME_STAMP_MASK ((1LLU << 56) - 1)
 269 #define DO_TIME_STAMP_MASK(ns64) (ns64 & TIME_STAMP_MASK)
 270
 271 /* TimeStampUnpack:
 272  * Unpack the timestamp value from the given COMMAND_TIMESTAMP command
 273  */
 274 static IMG_UINT64 TimeStampUnpack(COMMAND_TIMESTAMP *psTimeStamp)
 275 {
 276         IMG_UINT64 ui64TimeNs = 0;
 277         IMG_UINT32 i;
 278
 279         for(i = IMG_ARR_NUM_ELEMS(psTimeStamp->aui8TimeNs); i > 0; i--)
 280         {
 281                 ui64TimeNs <<= 8;
 282                 ui64TimeNs |= psTimeStamp->aui8TimeNs[i - 1];
 283         }
 284
 285         return ui64TimeNs;
 286 }
 287
 288 #if defined(PDUMP)
 289
 290 static void EmitPDumpAllocation(IMG_UINT32 ui32AllocationIndex,
 291                                         RECORD_ALLOCATION *psAlloc)
 292 {
 293         PDUMPCOMMENT("[SrvPFD] Allocation: %u"
 294                         " Addr: " IMG_DEV_VIRTADDR_FMTSPEC
 295                         " Size: " IMG_DEVMEM_SIZE_FMTSPEC
 296                         " Page size: %u"
 297                         " PID: %u"
 298                         " Process: %s"
 299                         " Name: %s",
 300                         ui32AllocationIndex,
 301                         psAlloc->sDevVAddr.uiAddr,
 302                         psAlloc->uiSize,
 303                         1U << psAlloc->ui32Log2PageSize,
 304                         psAlloc->uiPID,
 305                         OSGetCurrentClientProcessNameKM(),
 306                         psAlloc->szName);
 307 }
 308
 309 static void EmitPDumpMapUnmapAll(COMMAND_TYPE eType,
 310                                         IMG_UINT32 ui32AllocationIndex)
 311 {
 312         const IMG_CHAR *pszOpName;
 313
 314         switch(eType)
 315         {
 316                 case COMMAND_TYPE_MAP_ALL:
 317                         pszOpName = "MAP_ALL";
 318                         break;
 319                 case COMMAND_TYPE_UNMAP_ALL:
 320                         pszOpName = "UNMAP_ALL";
 321                         break;
 322                 default:
 323                         PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapAll: Invalid type: %u",
 324                                                                                 eType));
 325                         return;
 326
 327         }
 328
 329         PDUMPCOMMENT("[SrvPFD] Op: %s Allocation: %u",
 330                                                                 pszOpName,
 331                                                                 ui32AllocationIndex);
 332 }
 333
 334 static void EmitPDumpMapUnmapRange(COMMAND_TYPE eType,
 335                                         IMG_UINT32 ui32AllocationIndex,
 336                                         IMG_UINT32 ui32StartPage,
 337                                         IMG_UINT32 ui32Count)
 338 {
 339         const IMG_CHAR *pszOpName;
 340
 341         switch(eType)
 342         {
 343                 case COMMAND_TYPE_MAP_RANGE:
 344                         pszOpName = "MAP_RANGE";
 345                         break;
 346                 case COMMAND_TYPE_UNMAP_RANGE:
 347                         pszOpName = "UNMAP_RANGE";
 348                         break;
 349                 default:
 350                         PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapRange: Invalid type: %u",
 351                                                                                 eType));
 352                         return;
 353         }
 354
 355         PDUMPCOMMENT("[SrvPFD] Op: %s Allocation: %u Start Page: %u Count: %u",
 356                                                                         pszOpName,
 357                                                                         ui32AllocationIndex,
 358                                                                         ui32StartPage,
 359                                                                         ui32Count);
 360 }
 361
 362 #endif
 363
 364 /* InsertTimeStampCommand:
 365  * Insert a timestamp command into the circular buffer.
 366  */
 367 static void InsertTimeStampCommand(IMG_UINT64 ui64Now)
 368 {
 369         COMMAND_WRAPPER *psCommand;
 370
 371         psCommand = AcquireCBSlot();
 372
 373         psCommand->ui8Type = COMMAND_TYPE_TIMESTAMP;
 374
 375         TimeStampPack(&psCommand->u.sTimeStamp, ui64Now);
 376 }
 377
 378 /* InsertMapAllCommand:
 379  * Insert a "MAP_ALL" command for the given allocation into the circular buffer
 380  */
 381 static void InsertMapAllCommand(IMG_UINT32 ui32AllocIndex)
 382 {
 383         COMMAND_WRAPPER *psCommand;
 384
 385         psCommand = AcquireCBSlot();
 386
 387         psCommand->ui8Type = COMMAND_TYPE_MAP_ALL;
 388         psCommand->u.sMapAll.uiAllocIndex = ui32AllocIndex;
 389
 390 #if defined(PDUMP)
 391         EmitPDumpMapUnmapAll(COMMAND_TYPE_MAP_ALL, ui32AllocIndex);
 392 #endif
 393 }
 394
 395 /* InsertUnmapAllCommand:
 396  * Insert a "UNMAP_ALL" command for the given allocation into the circular buffer
 397  */
 398 static void InsertUnmapAllCommand(IMG_UINT32 ui32AllocIndex)
 399 {
 400         COMMAND_WRAPPER *psCommand;
 401
 402         psCommand = AcquireCBSlot();
 403
 404         psCommand->ui8Type = COMMAND_TYPE_UNMAP_ALL;
 405         psCommand->u.sUnmapAll.uiAllocIndex = ui32AllocIndex;
 406
 407 #if defined(PDUMP)
 408         EmitPDumpMapUnmapAll(COMMAND_TYPE_UNMAP_ALL, ui32AllocIndex);
 409 #endif
 410 }
 411
 412 /* MapRangePack:
 413  * Pack the given StartPage and Count values into the 40-bit representation
 414  * in the MAP_RANGE command.
 415  */
 416 static void MapRangePack(COMMAND_MAP_RANGE *psMapRange,
 417                                                 IMG_UINT32 ui32StartPage,
 418                                                 IMG_UINT32 ui32Count)
 419 {
 420         IMG_UINT64 ui64Data;
 421         IMG_UINT32 i;
 422
 423         /* we must encode the data into 40 bits:
 424          *   18 bits for the start page index
 425          *   12 bits for the range
 426         */
 427
 428         PVR_ASSERT(ui32StartPage <= MAP_RANGE_MAX_START);
 429         PVR_ASSERT(ui32Count <= MAP_RANGE_MAX_RANGE);
 430
 431         ui64Data = (((IMG_UINT64) ui32StartPage) << 12) | ui32Count;
 432
 433         for(i = 0; i < IMG_ARR_NUM_ELEMS(psMapRange->aui8Data); i++)
 434         {
 435                 psMapRange->aui8Data[i] = ui64Data & 0xFF;
 436                 ui64Data >>= 8;
 437         }
 438 }
 439
 440 /* MapRangePack:
 441  * Unpack the StartPage and Count values from the 40-bit representation
 442  * in the MAP_RANGE command.
 443  */
 444 static void MapRangeUnpack(COMMAND_MAP_RANGE *psMapRange,
 445                                                 IMG_UINT32 *pui32StartPage,
 446                                                 IMG_UINT32 *pui32Count)
 447 {
 448         IMG_UINT64 ui64Data = 0;
 449         IMG_UINT32 i;
 450
 451         for(i = IMG_ARR_NUM_ELEMS(psMapRange->aui8Data); i > 0; i--)
 452         {
 453                 ui64Data <<= 8;
 454                 ui64Data |= psMapRange->aui8Data[i - 1];
 455         }
 456
 457         *pui32StartPage = (ui64Data >> 12);
 458         *pui32Count = ui64Data & ((1 << 12) - 1);
 459 }
 460
 461 /* InsertMapRangeCommand:
 462  * Insert a MAP_RANGE command into the circular buffer with the given
 463  * StartPage and Count values.
 464  */
 465 static void InsertMapRangeCommand(IMG_UINT32 ui32AllocIndex,
 466                                                 IMG_UINT32 ui32StartPage,
 467                                                 IMG_UINT32 ui32Count)
 468 {
 469         COMMAND_WRAPPER *psCommand;
 470
 471         psCommand = AcquireCBSlot();
 472
 473         psCommand->ui8Type = COMMAND_TYPE_MAP_RANGE;
 474         psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
 475
 476         MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
 477
 478 #if defined(PDUMP)
 479         EmitPDumpMapUnmapRange(COMMAND_TYPE_MAP_RANGE,
 480                                                         ui32AllocIndex,
 481                                                         ui32StartPage,
 482                                                         ui32Count);
 483 #endif
 484 }
 485
 486 /* InsertUnmapRangeCommand:
 487  * Insert a UNMAP_RANGE command into the circular buffer with the given
 488  * StartPage and Count values.
 489  */
 490 static void InsertUnmapRangeCommand(IMG_UINT32 ui32AllocIndex,
 491                                                 IMG_UINT32 ui32StartPage,
 492                                                 IMG_UINT32 ui32Count)
 493 {
 494         COMMAND_WRAPPER *psCommand;
 495
 496         psCommand = AcquireCBSlot();
 497
 498         psCommand->ui8Type = COMMAND_TYPE_UNMAP_RANGE;
 499         psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
 500
 501         MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
 502
 503 #if defined(PDUMP)
 504         EmitPDumpMapUnmapRange(COMMAND_TYPE_UNMAP_RANGE,
 505                                                         ui32AllocIndex,
 506                                                         ui32StartPage,
 507                                                         ui32Count);
 508 #endif
 509 }
 510
 511 /* InsertAllocationToList:
 512  * Helper function for the allocation list.
 513  * Inserts the given allocation at the head of the list, whose current head is
 514  * pointed to by pui32ListHead
 515  */
 516 static void InsertAllocationToList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
 517 {
 518         RECORD_ALLOCATION *psAlloc;
 519
 520         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 521
 522         if(*pui32ListHead == END_OF_LIST)
 523         {
 524                 /* list is currently empty, so just replace it */
 525                 *pui32ListHead = ui32Alloc;
 526                 psAlloc->ui32Next = psAlloc->ui32Prev = *pui32ListHead;
 527         }
 528         else
 529         {
 530                 RECORD_ALLOCATION *psHeadAlloc;
 531                 RECORD_ALLOCATION *psTailAlloc;
 532
 533                 psHeadAlloc = ALLOC_INDEX_TO_PTR(*pui32ListHead);
 534                 psTailAlloc = ALLOC_INDEX_TO_PTR(psHeadAlloc->ui32Prev);
 535
 536                 /* make the new alloc point forwards to the previous head */
 537                 psAlloc->ui32Next = *pui32ListHead;
 538                 /* make the new alloc point backwards to the previous tail */
 539                 psAlloc->ui32Prev = psHeadAlloc->ui32Prev;
 540
 541                 /* the head is now our new alloc */
 542                 *pui32ListHead = ui32Alloc;
 543
 544                 /* the old head now points back to the new head */
 545                 psHeadAlloc->ui32Prev = *pui32ListHead;
 546
 547                 /* the tail now points forward to the new head */
 548                 psTailAlloc->ui32Next = ui32Alloc;
 549         }
 550 }
 551
 552 static void InsertAllocationToBusyList(IMG_UINT32 ui32Alloc)
 553 {
 554         InsertAllocationToList(&gsDevicememHistoryData.sRecords.ui32AllocationsListHead, ui32Alloc);
 555 }
 556
 557 /* RemoveAllocationFromList:
 558  * Helper function for the allocation list.
 559  * Removes the given allocation from the list, whose head is
 560  * pointed to by pui32ListHead
 561  */
 562 static void RemoveAllocationFromList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
 563 {
 564         RECORD_ALLOCATION *psAlloc;
 565
 566         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 567
 568         /* if this is the only element in the list then just make the list empty */
 569         if((*pui32ListHead == ui32Alloc) && (psAlloc->ui32Next == ui32Alloc))
 570         {
 571                 *pui32ListHead = END_OF_LIST;
 572         }
 573         else
 574         {
 575                 RECORD_ALLOCATION *psPrev, *psNext;
 576
 577                 psPrev = ALLOC_INDEX_TO_PTR(psAlloc->ui32Prev);
 578                 psNext = ALLOC_INDEX_TO_PTR(psAlloc->ui32Next);
 579
 580                 /* remove the allocation from the list */
 581                 psPrev->ui32Next = psAlloc->ui32Next;
 582                 psNext->ui32Prev = psAlloc->ui32Prev;
 583
 584                 /* if this allocation is the head then update the head */
 585                 if(*pui32ListHead == ui32Alloc)
 586                 {
 587                         *pui32ListHead = psAlloc->ui32Prev;
 588                 }
 589         }
 590 }
 591
 592 static void RemoveAllocationFromBusyList(IMG_UINT32 ui32Alloc)
 593 {
 594         RemoveAllocationFromList(&gsDevicememHistoryData.sRecords.ui32AllocationsListHead, ui32Alloc);
 595 }
 596
 597 /* TouchBusyAllocation:
 598  * Move the given allocation to the head of the list
 599  */
 600 static void TouchBusyAllocation(IMG_UINT32 ui32Alloc)
 601 {
 602         RemoveAllocationFromBusyList(ui32Alloc);
 603         InsertAllocationToBusyList(ui32Alloc);
 604 }
 605
 606 static INLINE IMG_BOOL IsAllocationListEmpty(IMG_UINT32 ui32ListHead)
 607 {
 608         return ui32ListHead == END_OF_LIST;
 609 }
 610
 611 /* GetOldestBusyAllocation:
 612  * Returns the index of the oldest allocation in the MRU list
 613  */
 614 static IMG_UINT32 GetOldestBusyAllocation(void)
 615 {
 616         IMG_UINT32 ui32Alloc;
 617         RECORD_ALLOCATION *psAlloc;
 618
 619         ui32Alloc = gsDevicememHistoryData.sRecords.ui32AllocationsListHead;
 620
 621         if(ui32Alloc == END_OF_LIST)
 622         {
 623                 return END_OF_LIST;
 624         }
 625
 626         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 627
 628         return psAlloc->ui32Prev;
 629 }
 630
 631 static IMG_UINT32 GetFreeAllocation(void)
 632 {
 633         IMG_UINT32 ui32Alloc;
 634
 635         ui32Alloc = GetOldestBusyAllocation();
 636
 637         return ui32Alloc;
 638 }
 639
 640 /* FindAllocation:
 641  * Searches the list of allocations and returns the index if an allocation
 642  * is found which matches the given properties
 643  */
 644 static IMG_UINT32 FindAllocation(const IMG_CHAR *pszName,
 645                                                         IMG_UINT64 ui64Serial,
 646                                                         IMG_PID uiPID,
 647                                                         IMG_DEV_VIRTADDR sDevVAddr,
 648                                                         IMG_DEVMEM_SIZE_T uiSize)
 649 {
 650         IMG_UINT32 ui32Head, ui32Index;
 651         RECORD_ALLOCATION *psAlloc;
 652
 653         ui32Head = ui32Index = gsDevicememHistoryData.sRecords.ui32AllocationsListHead;
 654
 655         if(IsAllocationListEmpty(ui32Index))
 656         {
 657                 goto not_found;
 658         }
 659
 660         do
 661         {
 662                 psAlloc = &gsDevicememHistoryData.sRecords.pasAllocations[ui32Index];
 663
 664                 if(     (psAlloc->ui64Serial == ui64Serial) &&
 665                         (psAlloc->sDevVAddr.uiAddr == sDevVAddr.uiAddr) &&
 666                         (psAlloc->uiSize == uiSize) &&
 667                         (strcmp(psAlloc->szName, pszName) == 0))
 668                 {
 669                         goto found;
 670                 }
 671
 672                 ui32Index = psAlloc->ui32Next;
 673         } while(ui32Index != ui32Head);
 674
 675 not_found:
 676         /* not found */
 677         ui32Index = END_OF_LIST;
 678
 679 found:
 680         /* if the allocation was not found then we return END_OF_LIST.
 681          * otherwise, we return the index of the allocation
 682          */
 683
 684         return ui32Index;
 685 }
 686
 687 /* InitialiseAllocation:
 688  * Initialise the given allocation structure with the given properties
 689  */
 690 static void InitialiseAllocation(RECORD_ALLOCATION *psAlloc,
 691                                                         const IMG_CHAR *pszName,
 692                                                         IMG_UINT64 ui64Serial,
 693                                                         IMG_PID uiPID,
 694                                                         IMG_DEV_VIRTADDR sDevVAddr,
 695                                                         IMG_DEVMEM_SIZE_T uiSize,
 696                                                         IMG_UINT32 ui32Log2PageSize)
 697 {
 698         OSStringNCopy(psAlloc->szName, pszName, sizeof(psAlloc->szName));
 699         psAlloc->szName[sizeof(psAlloc->szName) - 1] = '\0';
 700         psAlloc->ui64Serial = ui64Serial;
 701         psAlloc->uiPID = uiPID;
 702         psAlloc->sDevVAddr = sDevVAddr;
 703         psAlloc->uiSize = uiSize;
 704         psAlloc->ui32Log2PageSize = ui32Log2PageSize;
 705         psAlloc->ui64CreationTime = OSClockns64();
 706 }
 707
 708 /* CreateAllocation:
 709  * Creates a new allocation with the given properties then outputs the
 710  * index of the allocation
 711  */
 712 static PVRSRV_ERROR CreateAllocation(const IMG_CHAR *pszName,
 713                                                         IMG_UINT64 ui64Serial,
 714                                                         IMG_PID uiPID,
 715                                                         IMG_DEV_VIRTADDR sDevVAddr,
 716                                                         IMG_DEVMEM_SIZE_T uiSize,
 717                                                         IMG_UINT32 ui32Log2PageSize,
 718                                                         IMG_BOOL bAutoPurge,
 719                                                         IMG_UINT32 *puiAllocationIndex)
 720 {
 721         IMG_UINT32 ui32Alloc;
 722         RECORD_ALLOCATION *psAlloc;
 723
 724         ui32Alloc = GetFreeAllocation();
 725
 726         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 727
 728         InitialiseAllocation(ALLOC_INDEX_TO_PTR(ui32Alloc),
 729                                                 pszName,
 730                                                 ui64Serial,
 731                                                 uiPID,
 732                                                 sDevVAddr,
 733                                                 uiSize,
 734                                                 ui32Log2PageSize);
 735
 736         /* put the newly initialised allocation at the front of the MRU list */
 737         TouchBusyAllocation(ui32Alloc);
 738
 739         *puiAllocationIndex = ui32Alloc;
 740
 741 #if defined(PDUMP)
 742         EmitPDumpAllocation(ui32Alloc, psAlloc);
 743 #endif
 744
 745         return PVRSRV_OK;
 746 }
 747
 748 /* MatchAllocation:
 749  * Tests if the allocation at the given index matches the supplied properties.
 750  * Returns IMG_TRUE if it is a match, otherwise IMG_FALSE.
 751  */
 752 static IMG_BOOL MatchAllocation(IMG_UINT32 ui32AllocationIndex,
 753                                                 IMG_UINT64 ui64Serial,
 754                                                 IMG_DEV_VIRTADDR sDevVAddr,
 755                                                 IMG_DEVMEM_SIZE_T uiSize,
 756                                                 const IMG_CHAR *pszName,
 757                                                 IMG_UINT32 ui32Log2PageSize,
 758                                                 IMG_PID uiPID)
 759 {
 760         RECORD_ALLOCATION *psAlloc;
 761
 762         psAlloc = ALLOC_INDEX_TO_PTR(ui32AllocationIndex);
 763
 764         return  (psAlloc->ui64Serial == ui64Serial) &&
 765                         (psAlloc->sDevVAddr.uiAddr == sDevVAddr.uiAddr) &&
 766                         (psAlloc->uiSize == uiSize) &&
 767                         (psAlloc->ui32Log2PageSize == ui32Log2PageSize) &&
 768                         (strcmp(psAlloc->szName, pszName) == 0);
 769 }
 770
 771 /* FindOrCreateAllocation:
 772  * Convenience function.
 773  * Given a set of allocation properties (serial, DevVAddr, size, name, etc),
 774  * this function will look for an existing record of this allocation and
 775  * create the allocation if there is no existing record
 776  */
 777 static PVRSRV_ERROR FindOrCreateAllocation(IMG_UINT32 ui32AllocationIndexHint,
 778                                                         IMG_UINT64 ui64Serial,
 779                                                         IMG_DEV_VIRTADDR sDevVAddr,
 780                                                         IMG_DEVMEM_SIZE_T uiSize,
 781                                                         const char *pszName,
 782                                                         IMG_UINT32 ui32Log2PageSize,
 783                                                         IMG_PID uiPID,
 784                                                         IMG_BOOL bSparse,
 785                                                         IMG_UINT32 *pui32AllocationIndexOut,
 786                                                         IMG_BOOL *pbCreated)
 787 {
 788         IMG_UINT32 ui32AllocationIndex;
 789
 790         if(ui32AllocationIndexHint != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE)
 791         {
 792                 IMG_BOOL bHaveAllocation;
 793
 794                 /* first, try to match against the index given by the client */
 795                 bHaveAllocation = MatchAllocation(ui32AllocationIndexHint,
 796                                                                 ui64Serial,
 797                                                                 sDevVAddr,
 798                                                                 uiSize,
 799                                                                 pszName,
 800                                                                 ui32Log2PageSize,
 801                                                                 uiPID);
 802                 if(bHaveAllocation)
 803                 {
 804                         *pbCreated = IMG_FALSE;
 805                         *pui32AllocationIndexOut = ui32AllocationIndexHint;
 806                         return PVRSRV_OK;
 807                 }
 808         }
 809
 810         /* if matching against the client-supplied index fails then check
 811          * if the allocation exists in the list
 812          */
 813         ui32AllocationIndex = FindAllocation(pszName,
 814                                                 ui64Serial,
 815                                                 uiPID,
 816                                                 sDevVAddr,
 817                                                 uiSize);
 818
 819         /* if there is no record of the allocation then we
 820          * create it now
 821          */
 822         if(ui32AllocationIndex == END_OF_LIST)
 823         {
 824                 PVRSRV_ERROR eError;
 825                 eError = CreateAllocation(pszName,
 826                                                 ui64Serial,
 827                                                 uiPID,
 828                                                 sDevVAddr,
 829                                                 uiSize,
 830                                                 ui32Log2PageSize,
 831                                                 IMG_TRUE,
 832                                                 &ui32AllocationIndex);
 833
 834                 if(eError == PVRSRV_OK)
 835                 {
 836                         *pui32AllocationIndexOut = ui32AllocationIndex;
 837                         *pbCreated = IMG_TRUE;
 838                 }
 839                 else
 840                 {
 841                         PVR_DPF((PVR_DBG_ERROR,
 842                                 "%s: Failed to create record for allocation %s",
 843                                                                         __func__,
 844                                                                         pszName));
 845                 }
 846
 847                 return eError;
 848         }
 849         else
 850         {
 851                 /* found existing record */
 852                 *pui32AllocationIndexOut = ui32AllocationIndex;
 853                 *pbCreated = IMG_FALSE;
 854                 return PVRSRV_OK;
 855         }
 856
 857 }
 858
 859 /* GenerateMapUnmapCommandsForSparsePMR:
 860  * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the PMR's
 861  * current mapping table
 862  *
 863  * PMR: The PMR whose mapping table to read.
 864  * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
 865  * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
 866  *
 867  * This function goes through every page in the PMR's mapping table and looks for
 868  * virtually contiguous ranges to record as being mapped or unmapped.
 869  */
 870 static void GenerateMapUnmapCommandsForSparsePMR(PMR *psPMR,
 871                                                         IMG_UINT32 ui32AllocIndex,
 872                                                         IMG_BOOL bMap)
 873 {
 874         PMR_MAPPING_TABLE *psMappingTable;
 875         IMG_UINT32 ui32DonePages = 0;
 876         IMG_UINT32 ui32NumPages;
 877         IMG_UINT32 i;
 878         IMG_BOOL bInARun = IMG_FALSE;
 879         IMG_UINT32 ui32CurrentStart = 0;
 880         IMG_UINT32 ui32RunCount = 0;
 881
 882         psMappingTable = PMR_GetMappigTable(psPMR);
 883         ui32NumPages = psMappingTable->ui32NumPhysChunks;
 884
 885         if(ui32NumPages == 0)
 886         {
 887                 /* nothing to do */
 888                 return;
 889         }
 890
 891         for(i = 0; i < psMappingTable->ui32NumVirtChunks; i++)
 892         {
 893                 if(psMappingTable->aui32Translation[i] != TRANSLATION_INVALID)
 894                 {
 895                         if(!bInARun)
 896                         {
 897                                 bInARun = IMG_TRUE;
 898                                 ui32CurrentStart = i;
 899                                 ui32RunCount = 1;
 900                         }
 901                         else
 902                         {
 903                                 ui32RunCount++;
 904                         }
 905                 }
 906
 907                 if(bInARun)
 908                 {
 909                         /* test if we need to end this current run and generate the command,
 910                          * either because the next page is not virtually contiguous
 911                          * to the current page, we have reached the maximum range,
 912                          * or this is the last page in the mapping table
 913                          */
 914                         if((psMappingTable->aui32Translation[i] == TRANSLATION_INVALID) ||
 915                                                 (ui32RunCount == MAP_RANGE_MAX_RANGE) ||
 916                                                 (i == (psMappingTable->ui32NumVirtChunks - 1)))
 917                         {
 918                                 if(bMap)
 919                                 {
 920                                         InsertMapRangeCommand(ui32AllocIndex,
 921                                                                                 ui32CurrentStart,
 922                                                                                 ui32RunCount);
 923                                 }
 924                                 else
 925                                 {
 926                                         InsertUnmapRangeCommand(ui32AllocIndex,
 927                                                                                 ui32CurrentStart,
 928                                                                                 ui32RunCount);
 929                                 }
 930
 931                                 ui32DonePages += ui32RunCount;
 932
 933                                 if(ui32DonePages == ui32NumPages)
 934                                 {
 935                                          break;
 936                                 }
 937
 938                                 bInARun = IMG_FALSE;
 939                         }
 940                 }
 941         }
 942
 943 }
 944
 945 /* GenerateMapUnmapCommandsForChangeList:
 946  * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the
 947  * list of page change (page map or page unmap) indices given.
 948  *
 949  * ui32NumPages: Number of pages which have changed.
 950  * pui32PageList: List of indices of the pages which have changed.
 951  * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
 952  * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
 953  *
 954  * This function goes through every page in the list and looks for
 955  * virtually contiguous ranges to record as being mapped or unmapped.
 956  */
 957 static void GenerateMapUnmapCommandsForChangeList(IMG_UINT32 ui32NumPages,
 958                                                         IMG_UINT32 *pui32PageList,
 959                                                         IMG_UINT32 ui32AllocIndex,
 960                                                         IMG_BOOL bMap)
 961 {
 962         IMG_UINT32 i;
 963         IMG_BOOL bInARun = IMG_FALSE;
 964         IMG_UINT32 ui32CurrentStart = 0;
 965         IMG_UINT32 ui32RunCount = 0;
 966
 967         for(i = 0; i < ui32NumPages; i++)
 968         {
 969                 if(!bInARun)
 970                 {
 971                         bInARun = IMG_TRUE;
 972                         ui32CurrentStart = pui32PageList[i];
 973                 }
 974
 975                 ui32RunCount++;
 976
 977                  /* we flush if:
 978                  * - the next page in the list is not one greater than the current page
 979                  * - this is the last page in the list
 980                  * - we have reached the maximum range size
 981                  */
 982                 if((i == (ui32NumPages - 1)) ||
 983                         ((pui32PageList[i] + 1) != pui32PageList[i + 1]) ||
 984                         (ui32RunCount == MAP_RANGE_MAX_RANGE))
 985                 {
 986                         if(bMap)
 987                         {
 988                                 InsertMapRangeCommand(ui32AllocIndex,
 989                                                                         ui32CurrentStart,
 990                                                                         ui32RunCount);
 991                         }
 992                         else
 993                         {
 994                                 InsertUnmapRangeCommand(ui32AllocIndex,
 995                                                                         ui32CurrentStart,
 996                                                                         ui32RunCount);
 997                         }
 998
 999                         bInARun = IMG_FALSE;
1000                         ui32RunCount = 0;
1001                 }
1002         }
1003 }
1004
1005 /* DevicememHistoryMapKM:
1006  * Entry point for when an allocation is mapped into the MMU GPU
1007  *
1008  * psPMR: The PMR to which the allocation belongs.
1009  * ui32Offset: The offset within the PMR at which the allocation begins.
1010  * sDevVAddr: The DevVAddr at which the allocation begins.
1011  * szName: Annotation/name for the allocation.
1012  * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
1013  * ui32AllocationIndex: Allocation index as provided by the client.
1014  *                      We will use this as a short-cut to find the allocation
1015  *                      in our records.
1016  * pui32AllocationIndexOut: An updated allocation index for the client.
1017  *                          This may be a new value if we just created the
1018  *                          allocation record.
1019  */
1020 PVRSRV_ERROR DevicememHistoryMapNewKM(PMR *psPMR,
1021                                                         IMG_UINT32 ui32Offset,
1022                                                         IMG_DEV_VIRTADDR sDevVAddr,
1023                                                         IMG_DEVMEM_SIZE_T uiSize,
1024                                                         const char szName[DEVICEMEM_HISTORY_TEXT_BUFSZ],
1025                                                         IMG_UINT32 ui32Log2PageSize,
1026                                                         IMG_UINT32 ui32AllocationIndex,
1027                                                         IMG_UINT32 *pui32AllocationIndexOut)
1028 {
1029         IMG_BOOL bSparse = PMR_IsSparse(psPMR);
1030         IMG_UINT64 ui64Serial;
1031         IMG_PID uiPID = OSGetCurrentProcessID();
1032         PVRSRV_ERROR eError;
1033         IMG_BOOL bCreated;
1034
1035         if((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1036                                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1037         {
1038                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1039                                                                 __func__,
1040                                                                 ui32AllocationIndex));
1041                 return PVRSRV_ERROR_INVALID_PARAMS;
1042         }
1043
1044         PMRGetUID(psPMR, &ui64Serial);
1045
1046         DevicememHistoryLock();
1047
1048         eError = FindOrCreateAllocation(ui32AllocationIndex,
1049                                                 ui64Serial,
1050                                                 sDevVAddr,
1051                                                 uiSize,
1052                                                 szName,
1053                                                 ui32Log2PageSize,
1054                                                 uiPID,
1055                                                 bSparse,
1056                                                 &ui32AllocationIndex,
1057                                                 &bCreated);
1058
1059         if((eError == PVRSRV_OK) && !bCreated)
1060         {
1061                 /* touch the allocation so it goes to the head of our MRU list */
1062                 TouchBusyAllocation(ui32AllocationIndex);
1063         }
1064         else if(eError != PVRSRV_OK)
1065         {
1066                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1067                                                                         __func__,
1068                                                                         szName,
1069                                                                         PVRSRVGETERRORSTRING(eError)));
1070                 goto out_unlock;
1071         }
1072
1073         if(!bSparse)
1074         {
1075                 InsertMapAllCommand(ui32AllocationIndex);
1076         }
1077         else
1078         {
1079                 GenerateMapUnmapCommandsForSparsePMR(psPMR,
1080                                                                 ui32AllocationIndex,
1081                                                                 IMG_TRUE);
1082         }
1083
1084         InsertTimeStampCommand(OSClockns64());
1085
1086         *pui32AllocationIndexOut = ui32AllocationIndex;
1087
1088 out_unlock:
1089         DevicememHistoryUnlock();
1090
1091         return eError;
1092 }
1093
1094 static void VRangeInsertMapUnmapCommands(IMG_BOOL bMap,
1095                                                         IMG_UINT32 ui32AllocationIndex,
1096                                                         IMG_DEV_VIRTADDR sBaseDevVAddr,
1097                                                         IMG_UINT32 ui32StartPage,
1098                                                         IMG_UINT32 ui32NumPages,
1099                                                         const IMG_CHAR *pszName)
1100 {
1101         while(ui32NumPages > 0)
1102         {
1103                 IMG_UINT32 ui32PagesToAdd;
1104
1105                 ui32PagesToAdd = MIN(ui32NumPages, MAP_RANGE_MAX_RANGE);
1106
1107                 if(ui32StartPage > MAP_RANGE_MAX_START)
1108                 {
1109                         PVR_DPF((PVR_DBG_WARNING, "Cannot record %s range beginning at page "
1110                                                                         "%u on allocation %s",
1111                                                                         bMap ? "map" : "unmap",
1112                                                                         ui32StartPage,
1113                                                                         pszName));
1114                         return;
1115                 }
1116
1117                 if(bMap)
1118                 {
1119                         InsertMapRangeCommand(ui32AllocationIndex,
1120                                                                 ui32StartPage,
1121                                                                 ui32PagesToAdd);
1122                 }
1123                 else
1124                 {
1125                         InsertUnmapRangeCommand(ui32AllocationIndex,
1126                                                                 ui32StartPage,
1127                                                                 ui32PagesToAdd);
1128                 }
1129
1130                 ui32StartPage += ui32PagesToAdd;
1131                 ui32NumPages -= ui32PagesToAdd;
1132         }
1133 }
1134
1135 PVRSRV_ERROR DevicememHistoryMapVRangeKM(IMG_DEV_VIRTADDR sBaseDevVAddr,
1136                                                 IMG_UINT32 ui32StartPage,
1137                                                 IMG_UINT32 ui32NumPages,
1138                                                 IMG_DEVMEM_SIZE_T uiAllocSize,
1139                                                 const IMG_CHAR szName[DEVICEMEM_HISTORY_TEXT_BUFSZ],
1140                                                 IMG_UINT32 ui32Log2PageSize,
1141                                                 IMG_UINT32 ui32AllocationIndex,
1142                                                 IMG_UINT32 *pui32AllocationIndexOut)
1143 {
1144         IMG_PID uiPID = OSGetCurrentProcessID();
1145         PVRSRV_ERROR eError;
1146         IMG_BOOL bCreated;
1147
1148         if((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1149                                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1150         {
1151                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1152                                                                 __func__,
1153                                                         ui32AllocationIndex));
1154                 return PVRSRV_ERROR_INVALID_PARAMS;
1155         }
1156
1157         DevicememHistoryLock();
1158
1159         eError = FindOrCreateAllocation(ui32AllocationIndex,
1160                                                 0,
1161                                                 sBaseDevVAddr,
1162                                                 uiAllocSize,
1163                                                 szName,
1164                                                 ui32Log2PageSize,
1165                                                 uiPID,
1166                                                 IMG_FALSE,
1167                                                 &ui32AllocationIndex,
1168                                                 &bCreated);
1169
1170         if((eError == PVRSRV_OK) && !bCreated)
1171         {
1172                 /* touch the allocation so it goes to the head of our MRU list */
1173                 TouchBusyAllocation(ui32AllocationIndex);
1174         }
1175         else if(eError != PVRSRV_OK)
1176         {
1177                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1178                                                                         __func__,
1179                                                                         szName,
1180                                                                         PVRSRVGETERRORSTRING(eError)));
1181                 goto out_unlock;
1182         }
1183
1184         VRangeInsertMapUnmapCommands(IMG_TRUE,
1185                                                 ui32AllocationIndex,
1186                                                 sBaseDevVAddr,
1187                                                 ui32StartPage,
1188                                                 ui32NumPages,
1189                                                 szName);
1190
1191         *pui32AllocationIndexOut = ui32AllocationIndex;
1192
1193 out_unlock:
1194         DevicememHistoryUnlock();
1195
1196         return eError;
1197
1198 }
1199
1200 PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(IMG_DEV_VIRTADDR sBaseDevVAddr,
1201                                                 IMG_UINT32 ui32StartPage,
1202                                                 IMG_UINT32 ui32NumPages,
1203                                                 IMG_DEVMEM_SIZE_T uiAllocSize,
1204                                                 const IMG_CHAR szName[DEVICEMEM_HISTORY_TEXT_BUFSZ],
1205                                                 IMG_UINT32 ui32Log2PageSize,
1206                                                 IMG_UINT32 ui32AllocationIndex,
1207                                                 IMG_UINT32 *pui32AllocationIndexOut)
1208 {
1209         IMG_PID uiPID = OSGetCurrentProcessID();
1210         PVRSRV_ERROR eError;
1211         IMG_BOOL bCreated;
1212
1213         if((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1214                                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1215         {
1216                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1217                                                                 __func__,
1218                                                         ui32AllocationIndex));
1219                 return PVRSRV_ERROR_INVALID_PARAMS;
1220         }
1221
1222         DevicememHistoryLock();
1223
1224         eError = FindOrCreateAllocation(ui32AllocationIndex,
1225                                                 0,
1226                                                 sBaseDevVAddr,
1227                                                 uiAllocSize,
1228                                                 szName,
1229                                                 ui32Log2PageSize,
1230                                                 uiPID,
1231                                                 IMG_FALSE,
1232                                                 &ui32AllocationIndex,
1233                                                 &bCreated);
1234
1235         if((eError == PVRSRV_OK) && !bCreated)
1236         {
1237                 /* touch the allocation so it goes to the head of our MRU list */
1238                 TouchBusyAllocation(ui32AllocationIndex);
1239         }
1240         else if(eError != PVRSRV_OK)
1241         {
1242                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1243                                                                         __func__,
1244                                                                         szName,
1245                                                                         PVRSRVGETERRORSTRING(eError)));
1246                 goto out_unlock;
1247         }
1248
1249         VRangeInsertMapUnmapCommands(IMG_FALSE,
1250                                                 ui32AllocationIndex,
1251                                                 sBaseDevVAddr,
1252                                                 ui32StartPage,
1253                                                 ui32NumPages,
1254                                                 szName);
1255
1256         *pui32AllocationIndexOut = ui32AllocationIndex;
1257
1258 out_unlock:
1259         DevicememHistoryUnlock();
1260
1261         return eError;
1262 }
1263
1264
1265
1266 /* DevicememHistoryUnmapKM:
1267  * Entry point for when an allocation is unmapped from the MMU GPU
1268  *
1269  * psPMR: The PMR to which the allocation belongs.
1270  * ui32Offset: The offset within the PMR at which the allocation begins.
1271  * sDevVAddr: The DevVAddr at which the allocation begins.
1272  * szName: Annotation/name for the allocation.
1273  * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
1274  * ui32AllocationIndex: Allocation index as provided by the client.
1275  *                      We will use this as a short-cut to find the allocation
1276  *                      in our records.
1277  * pui32AllocationIndexOut: An updated allocation index for the client.
1278  *                          This may be a new value if we just created the
1279  *                          allocation record.
1280  */
1281 PVRSRV_ERROR DevicememHistoryUnmapNewKM(PMR *psPMR,
1282                                                         IMG_UINT32 ui32Offset,
1283                                                         IMG_DEV_VIRTADDR sDevVAddr,
1284                                                         IMG_DEVMEM_SIZE_T uiSize,
1285                                                         const char szName[DEVICEMEM_HISTORY_TEXT_BUFSZ],
1286                                                         IMG_UINT32 ui32Log2PageSize,
1287                                                         IMG_UINT32 ui32AllocationIndex,
1288                                                         IMG_UINT32 *pui32AllocationIndexOut)
1289 {
1290         IMG_BOOL bSparse = PMR_IsSparse(psPMR);
1291         IMG_UINT64 ui64Serial;
1292         IMG_PID uiPID = OSGetCurrentProcessID();
1293         PVRSRV_ERROR eError;
1294         IMG_BOOL bCreated;
1295
1296         if((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1297                                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1298         {
1299                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1300                                                                 __func__,
1301                                                                 ui32AllocationIndex));
1302                 return PVRSRV_ERROR_INVALID_PARAMS;
1303         }
1304
1305         PMRGetUID(psPMR, &ui64Serial);
1306
1307         DevicememHistoryLock();
1308
1309         eError = FindOrCreateAllocation(ui32AllocationIndex,
1310                                                 ui64Serial,
1311                                                 sDevVAddr,
1312                                                 uiSize,
1313                                                 szName,
1314                                                 ui32Log2PageSize,
1315                                                 uiPID,
1316                                                 bSparse,
1317                                                 &ui32AllocationIndex,
1318                                                 &bCreated);
1319
1320         if((eError == PVRSRV_OK) && !bCreated)
1321         {
1322                 /* touch the allocation so it goes to the head of our MRU list */
1323                 TouchBusyAllocation(ui32AllocationIndex);
1324         }
1325         else if(eError != PVRSRV_OK)
1326         {
1327                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1328                                                                         __func__,
1329                                                                         szName,
1330                                                                         PVRSRVGETERRORSTRING(eError)));
1331                 goto out_unlock;
1332         }
1333
1334         if(!bSparse)
1335         {
1336                 InsertUnmapAllCommand(ui32AllocationIndex);
1337         }
1338         else
1339         {
1340                 GenerateMapUnmapCommandsForSparsePMR(psPMR,
1341                                                                 ui32AllocationIndex,
1342                                                                 IMG_FALSE);
1343         }
1344
1345         InsertTimeStampCommand(OSClockns64());
1346
1347         *pui32AllocationIndexOut = ui32AllocationIndex;
1348
1349 out_unlock:
1350         DevicememHistoryUnlock();
1351
1352         return eError;
1353 }
1354
1355 /* DevicememHistorySparseChangeKM:
1356  * Entry point for when a sparse allocation is changed, such that some of the
1357  * pages within the sparse allocation are mapped or unmapped.
1358  *
1359  * psPMR: The PMR to which the allocation belongs.
1360  * ui32Offset: The offset within the PMR at which the allocation begins.
1361  * sDevVAddr: The DevVAddr at which the allocation begins.
1362  * szName: Annotation/name for the allocation.
1363  * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
1364  * ui32AllocPageCount: Number of pages which have been mapped.
1365  * paui32AllocPageIndices: Indices of pages which have been mapped.
1366  * ui32FreePageCount: Number of pages which have been unmapped.
1367  * paui32FreePageIndices: Indices of pages which have been unmapped.
1368  * ui32AllocationIndex: Allocation index as provided by the client.
1369  *                      We will use this as a short-cut to find the allocation
1370  *                      in our records.
1371  * pui32AllocationIndexOut: An updated allocation index for the client.
1372  *                          This may be a new value if we just created the
1373  *                          allocation record.
1374  */
1375 PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
1376                                                         IMG_UINT32 ui32Offset,
1377                                                         IMG_DEV_VIRTADDR sDevVAddr,
1378                                                         IMG_DEVMEM_SIZE_T uiSize,
1379                                                         const char szName[DEVICEMEM_HISTORY_TEXT_BUFSZ],
1380                                                         IMG_UINT32 ui32Log2PageSize,
1381                                                         IMG_UINT32 ui32AllocPageCount,
1382                                                         IMG_UINT32 *paui32AllocPageIndices,
1383                                                         IMG_UINT32 ui32FreePageCount,
1384                                                         IMG_UINT32 *paui32FreePageIndices,
1385                                                         IMG_UINT32 ui32AllocationIndex,
1386                                                         IMG_UINT32 *pui32AllocationIndexOut)
1387 {
1388         IMG_UINT64 ui64Serial;
1389         IMG_PID uiPID = OSGetCurrentProcessID();
1390         PVRSRV_ERROR eError;
1391         IMG_BOOL bCreated;
1392
1393         if((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1394                                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1395         {
1396                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1397                                                                 __func__,
1398                                                                 ui32AllocationIndex));
1399                 return PVRSRV_ERROR_INVALID_PARAMS;
1400         }
1401
1402         PMRGetUID(psPMR, &ui64Serial);
1403
1404         DevicememHistoryLock();
1405
1406         eError = FindOrCreateAllocation(ui32AllocationIndex,
1407                                                 ui64Serial,
1408                                                 sDevVAddr,
1409                                                 uiSize,
1410                                                 szName,
1411                                                 ui32Log2PageSize,
1412                                                 uiPID,
1413                                                 IMG_TRUE /* bSparse */,
1414                                                 &ui32AllocationIndex,
1415                                                 &bCreated);
1416
1417         if((eError == PVRSRV_OK) && !bCreated)
1418         {
1419                 /* touch the allocation so it goes to the head of our MRU list */
1420                 TouchBusyAllocation(ui32AllocationIndex);
1421         }
1422         else if(eError != PVRSRV_OK)
1423         {
1424                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1425                                                                         __func__,
1426                                                                         szName,
1427                                                                         PVRSRVGETERRORSTRING(eError)));
1428                 goto out_unlock;
1429         }
1430
1431         GenerateMapUnmapCommandsForChangeList(ui32AllocPageCount,
1432                                                         paui32AllocPageIndices,
1433                                                         ui32AllocationIndex,
1434                                                         IMG_TRUE);
1435
1436         GenerateMapUnmapCommandsForChangeList(ui32FreePageCount,
1437                                                         paui32FreePageIndices,
1438                                                         ui32AllocationIndex,
1439                                                         IMG_FALSE);
1440
1441         InsertTimeStampCommand(OSClockns64());
1442
1443         *pui32AllocationIndexOut = ui32AllocationIndex;
1444
1445 out_unlock:
1446         DevicememHistoryUnlock();
1447
1448         return eError;
1449
1450 }
1451
1452 /* CircularBufferIterateStart:
1453  * Initialise local state for iterating over the circular buffer
1454  */
1455 static void CircularBufferIterateStart(IMG_UINT32 *pui32Head, IMG_UINT32 *pui32Iter)
1456 {
1457         *pui32Head = gsDevicememHistoryData.sRecords.ui32Head;
1458
1459         if(*pui32Head != 0)
1460         {
1461                 *pui32Iter = *pui32Head - 1;
1462         }
1463         else
1464         {
1465                 *pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
1466         }
1467 }
1468
1469 /* CircularBufferIteratePrevious:
1470  * Iterate to the previous item in the circular buffer.
1471  * This is called repeatedly to iterate over the whole circular buffer.
1472  */
1473 static COMMAND_WRAPPER *CircularBufferIteratePrevious(IMG_UINT32 ui32Head,
1474                                                         IMG_UINT32 *pui32Iter,
1475                                                         COMMAND_TYPE *peType,
1476                                                         IMG_BOOL *pbLast)
1477 {
1478         IMG_UINT8 *pui8Header;
1479         COMMAND_WRAPPER *psOut = NULL;
1480
1481         psOut = gsDevicememHistoryData.sRecords.pasCircularBuffer + *pui32Iter;
1482
1483         pui8Header = (IMG_UINT8 *) psOut;
1484
1485         /* sanity check the command looks valid.
1486          * this condition should never happen, but check for it anyway
1487          * and try to handle it
1488          */
1489         if(*pui8Header >= COMMAND_TYPE_COUNT)
1490         {
1491                 /* invalid header detected. Circular buffer corrupted? */
1492                 PVR_DPF((PVR_DBG_ERROR, "CircularBufferIteratePrevious: "
1493                                                         "Invalid header: %u",
1494                                                         *pui8Header));
1495                 *pbLast = IMG_TRUE;
1496                 return NULL;
1497         }
1498
1499         *peType = *pui8Header;
1500
1501         if(*pui32Iter != 0)
1502         {
1503                 (*pui32Iter)--;
1504         }
1505         else
1506         {
1507                 *pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
1508         }
1509
1510
1511         /* inform the caller this is the last command if either we have reached
1512          * the head (where we started) or if we have reached an empty command,
1513          * which means we have covered all populated entries
1514          */
1515         if((*pui32Iter == ui32Head) || (*peType == COMMAND_TYPE_NONE))
1516         {
1517                 /* this is the final iteration */
1518                 *pbLast = IMG_TRUE;
1519         }
1520
1521         return psOut;
1522 }
1523
1524 /* MapUnmapCommandGetInfo:
1525  * Helper function to get the address and mapping information from a MAP_ALL, UNMAP_ALL,
1526  * MAP_RANGE or UNMAP_RANGE command
1527  */
1528 static void MapUnmapCommandGetInfo(COMMAND_WRAPPER *psCommand,
1529                                         COMMAND_TYPE eType,
1530                                         IMG_DEV_VIRTADDR *psDevVAddrStart,
1531                                         IMG_DEV_VIRTADDR *psDevVAddrEnd,
1532                                         IMG_BOOL *pbMap,
1533                                         IMG_UINT32 *pui32AllocIndex)
1534 {
1535         if((eType == COMMAND_TYPE_MAP_ALL) || ((eType == COMMAND_TYPE_UNMAP_ALL)))
1536         {
1537                 COMMAND_MAP_ALL *psMapAll = &psCommand->u.sMapAll;
1538                 RECORD_ALLOCATION *psAlloc;
1539
1540                 *pbMap = (eType == COMMAND_TYPE_MAP_ALL);
1541                 *pui32AllocIndex = psMapAll->uiAllocIndex;
1542
1543                 psAlloc = ALLOC_INDEX_TO_PTR(psMapAll->uiAllocIndex);
1544
1545                 *psDevVAddrStart = psAlloc->sDevVAddr;
1546                 psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr + psAlloc->uiSize - 1;
1547         }
1548         else if((eType == COMMAND_TYPE_MAP_RANGE) || ((eType == COMMAND_TYPE_UNMAP_RANGE)))
1549         {
1550                 COMMAND_MAP_RANGE *psMapRange = &psCommand->u.sMapRange;
1551                 RECORD_ALLOCATION *psAlloc;
1552                 IMG_UINT32 ui32StartPage, ui32Count;
1553
1554                 *pbMap = (eType == COMMAND_TYPE_MAP_RANGE);
1555                 *pui32AllocIndex = psMapRange->uiAllocIndex;
1556
1557                 psAlloc = ALLOC_INDEX_TO_PTR(psMapRange->uiAllocIndex);
1558
1559                 MapRangeUnpack(psMapRange, &ui32StartPage, &ui32Count);
1560
1561                 psDevVAddrStart->uiAddr = psAlloc->sDevVAddr.uiAddr +
1562                                 ((1U << psAlloc->ui32Log2PageSize) * ui32StartPage);
1563
1564                 psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr +
1565                                 ((1U << psAlloc->ui32Log2PageSize) * ui32Count) - 1;
1566         }
1567         else
1568         {
1569                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid command type: %u",
1570                                                                 __func__,
1571                                                                 eType));
1572         }
1573 }
1574
1575 /* DevicememHistoryQuery:
1576  * Entry point for rgxdebug to look up addresses relating to a page fault
1577  */
1578 IMG_BOOL DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
1579                                DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
1580                                IMG_UINT32 ui32PageSizeBytes,
1581                                IMG_BOOL bMatchAnyAllocInPage)
1582 {
1583         IMG_UINT32 ui32Head, ui32Iter;
1584         COMMAND_TYPE eType = COMMAND_TYPE_NONE;
1585         COMMAND_WRAPPER *psCommand = NULL;
1586         IMG_BOOL bLast = IMG_FALSE;
1587         IMG_UINT64 ui64StartTime = OSClockns64();
1588         IMG_UINT64 ui64TimeNs = 0;
1589
1590         /* initialise the results count for the caller */
1591         psQueryOut->ui32NumResults = 0;
1592
1593         DevicememHistoryLock();
1594
1595         /* if the search is constrained to a particular PID then we
1596          * first search the list of allocations to see if this
1597          * PID is known to us
1598          */
1599         if(psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY)
1600         {
1601                 IMG_UINT32 ui32Alloc;
1602                 ui32Alloc = gsDevicememHistoryData.sRecords.ui32AllocationsListHead;
1603
1604                 while(ui32Alloc != END_OF_LIST)
1605                 {
1606                         RECORD_ALLOCATION *psAlloc;
1607
1608                         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
1609
1610                         if(psAlloc->uiPID == psQueryIn->uiPID)
1611                         {
1612                                 goto found_pid;
1613                         }
1614
1615                         if(ui32Alloc == gsDevicememHistoryData.sRecords.ui32AllocationsListHead)
1616                         {
1617                                 /* gone through whole list */
1618                                 break;
1619                         }
1620                 }
1621
1622                 /* PID not found, so we do not have any suitable data for this
1623                  * page fault
1624                  */
1625                  goto out_unlock;
1626         }
1627
1628 found_pid:
1629
1630         CircularBufferIterateStart(&ui32Head, &ui32Iter);
1631
1632         while(!bLast)
1633         {
1634                 psCommand = CircularBufferIteratePrevious(ui32Head, &ui32Iter, &eType, &bLast);
1635
1636                 if(eType == COMMAND_TYPE_TIMESTAMP)
1637                 {
1638                         ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
1639                         continue;
1640                 }
1641
1642                 if((eType == COMMAND_TYPE_MAP_ALL) ||
1643                         (eType == COMMAND_TYPE_UNMAP_ALL) ||
1644                         (eType == COMMAND_TYPE_MAP_RANGE) ||
1645                         (eType == COMMAND_TYPE_UNMAP_RANGE))
1646                 {
1647                         RECORD_ALLOCATION *psAlloc;
1648                         IMG_DEV_VIRTADDR sAllocStartAddrOrig, sAllocEndAddrOrig;
1649                         IMG_DEV_VIRTADDR sAllocStartAddr, sAllocEndAddr;
1650                         IMG_BOOL bMap;
1651                         IMG_UINT32 ui32AllocIndex;
1652
1653                         MapUnmapCommandGetInfo(psCommand,
1654                                                         eType,
1655                                                         &sAllocStartAddrOrig,
1656                                                         &sAllocEndAddrOrig,
1657                                                         &bMap,
1658                                                         &ui32AllocIndex);
1659
1660                         sAllocStartAddr = sAllocStartAddrOrig;
1661                         sAllocEndAddr = sAllocEndAddrOrig;
1662
1663                         psAlloc = ALLOC_INDEX_TO_PTR(ui32AllocIndex);
1664
1665                         /* skip this command if we need to search within
1666                          * a particular PID, and this allocation is not from
1667                          * that PID
1668                          */
1669                         if((psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY) &&
1670                                 (psAlloc->uiPID != psQueryIn->uiPID))
1671                         {
1672                                 continue;
1673                         }
1674
1675                         /* if the allocation was created after this event, then this
1676                          * event must be for an old/removed allocation, so skip it
1677                          */
1678                         if(DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
1679                         {
1680                                 continue;
1681                         }
1682
1683                         /* if the caller wants us to match any allocation in the
1684                          * same page as the allocation then tweak the real start/end
1685                          * addresses of the allocation here
1686                          */
1687                         if(bMatchAnyAllocInPage)
1688                         {
1689                                 sAllocStartAddr.uiAddr = sAllocStartAddr.uiAddr & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
1690                                 sAllocEndAddr.uiAddr = (sAllocEndAddr.uiAddr + ui32PageSizeBytes - 1) & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
1691                         }
1692
1693                         if((psQueryIn->sDevVAddr.uiAddr >= sAllocStartAddr.uiAddr) &&
1694                                 (psQueryIn->sDevVAddr.uiAddr <  sAllocEndAddr.uiAddr))
1695                         {
1696                                 DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult = &psQueryOut->sResults[psQueryOut->ui32NumResults];
1697
1698                                 OSStringNCopy(psResult->szString, psAlloc->szName, sizeof(psResult->szString));
1699                                 psResult->szString[DEVICEMEM_HISTORY_TEXT_BUFSZ - 1] = '\0';
1700                                 psResult->sBaseDevVAddr = psAlloc->sDevVAddr;
1701                                 psResult->uiSize = psAlloc->uiSize;
1702                                 psResult->bMap = bMap;
1703                                 psResult->ui64Age = _CalculateAge(ui64StartTime, ui64TimeNs, TIME_STAMP_MASK);
1704                                 psResult->ui64When = ui64TimeNs;
1705                                 /* write the responsible PID in the placeholder */
1706                                 psResult->sProcessInfo.uiPID = psAlloc->uiPID;
1707
1708                                 if((eType == COMMAND_TYPE_MAP_ALL) || (eType == COMMAND_TYPE_UNMAP_ALL))
1709                                 {
1710                                         psResult->bRange = IMG_FALSE;
1711                                         psResult->bAll = IMG_TRUE;
1712                                 }
1713                                 else
1714                                 {
1715                                         psResult->bRange = IMG_TRUE;
1716                                         MapRangeUnpack(&psCommand->u.sMapRange,
1717                                                                                 &psResult->ui32StartPage,
1718                                                                                 &psResult->ui32PageCount);
1719                                         psResult->bAll = (psResult->ui32PageCount * (1U << psAlloc->ui32Log2PageSize))
1720                                                                                         == psAlloc->uiSize;
1721                                         psResult->sMapStartAddr = sAllocStartAddrOrig;
1722                                         psResult->sMapEndAddr = sAllocEndAddrOrig;
1723                                 }
1724
1725                                 psQueryOut->ui32NumResults++;
1726
1727                                 if(psQueryOut->ui32NumResults == DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS)
1728                                 {
1729                                         break;
1730                                 }
1731                         }
1732                 }
1733         }
1734
1735 out_unlock:
1736         DevicememHistoryUnlock();
1737
1738         return psQueryOut->ui32NumResults > 0;
1739 }
1740
1741 static void DeviceMemHistoryFmt(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN],
1742                                                         IMG_PID uiPID,
1743                                                         const IMG_CHAR *pszName,
1744                                                         const IMG_CHAR *pszAction,
1745                                                         IMG_DEV_VIRTADDR sDevVAddrStart,
1746                                                         IMG_DEV_VIRTADDR sDevVAddrEnd,
1747                                                         IMG_UINT64 ui64TimeNs)
1748 {
1749
1750         szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN - 1] = '\0';
1751         OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
1752                                 /* PID NAME MAP/UNMAP MIN-MAX SIZE AbsUS AgeUS*/
1753                                 "%04u %-40s %-10s "
1754                                 IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC " "
1755                                 "0x%08llX "
1756                                 "%013llu", /* 13 digits is over 2 hours of ns */
1757                                 uiPID,
1758                                 pszName,
1759                                 pszAction,
1760                                 sDevVAddrStart.uiAddr,
1761                                 sDevVAddrEnd.uiAddr,
1762                                 sDevVAddrEnd.uiAddr - sDevVAddrStart.uiAddr,
1763                                 ui64TimeNs);
1764 }
1765
1766 static void DeviceMemHistoryFmtHeader(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN])
1767 {
1768         OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
1769                                 "%-4s %-40s %-6s   %10s   %10s   %8s %13s",
1770                                 "PID",
1771                                 "NAME",
1772                                 "ACTION",
1773                                 "ADDR MIN",
1774                                 "ADDR MAX",
1775                                 "SIZE",
1776                                 "ABS NS");
1777 }
1778
1779 static const char *CommandTypeToString(COMMAND_TYPE eType)
1780 {
1781         switch(eType)
1782         {
1783                 case COMMAND_TYPE_MAP_ALL:
1784                         return "MapAll";
1785                 case COMMAND_TYPE_UNMAP_ALL:
1786                         return "UnmapAll";
1787                 case COMMAND_TYPE_MAP_RANGE:
1788                         return "MapRange";
1789                 case COMMAND_TYPE_UNMAP_RANGE:
1790                         return "UnmapRange";
1791                 case COMMAND_TYPE_TIMESTAMP:
1792                         return "TimeStamp";
1793                 default:
1794                         return "???";
1795         }
1796 }
1797
1798 static void DevicememHistoryPrintAll(void *pvFilePtr, OS_STATS_PRINTF_FUNC* pfnOSStatsPrintf)
1799 {
1800         IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
1801         IMG_UINT32 ui32Iter;
1802         IMG_UINT32 ui32Head;
1803         IMG_BOOL bLast = IMG_FALSE;
1804         IMG_UINT64 ui64TimeNs = 0;
1805         IMG_UINT64 ui64StartTime = OSClockns64();
1806
1807         DeviceMemHistoryFmtHeader(szBuffer);
1808         pfnOSStatsPrintf(pvFilePtr, "%s\n", szBuffer);
1809
1810         CircularBufferIterateStart(&ui32Head, &ui32Iter);
1811
1812         while(!bLast)
1813         {
1814                 COMMAND_WRAPPER *psCommand;
1815                 COMMAND_TYPE eType = COMMAND_TYPE_NONE;
1816
1817                 psCommand = CircularBufferIteratePrevious(ui32Head, &ui32Iter, &eType, &bLast);
1818
1819                 if(eType == COMMAND_TYPE_TIMESTAMP)
1820                 {
1821                         ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
1822                         continue;
1823                 }
1824
1825
1826                 if((eType == COMMAND_TYPE_MAP_ALL) ||
1827                         (eType == COMMAND_TYPE_UNMAP_ALL) ||
1828                         (eType == COMMAND_TYPE_MAP_RANGE) ||
1829                         (eType == COMMAND_TYPE_UNMAP_RANGE))
1830                 {
1831                         RECORD_ALLOCATION *psAlloc;
1832                         IMG_DEV_VIRTADDR sDevVAddrStart, sDevVAddrEnd;
1833                         IMG_BOOL bMap;
1834                         IMG_UINT32 ui32AllocIndex;
1835
1836                         MapUnmapCommandGetInfo(psCommand,
1837                                                                 eType,
1838                                                                 &sDevVAddrStart,
1839                                                                 &sDevVAddrEnd,
1840                                                                 &bMap,
1841                                                                 &ui32AllocIndex);
1842
1843                         psAlloc = ALLOC_INDEX_TO_PTR(ui32AllocIndex);
1844
1845                         if(DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
1846                         {
1847                                 /* if this event relates to an allocation we
1848                                  * are no longer tracking then do not print it
1849                                  */
1850                                 continue;
1851                         }
1852
1853                         DeviceMemHistoryFmt(szBuffer,
1854                                                                 psAlloc->uiPID,
1855                                                                 psAlloc->szName,
1856                                                                 CommandTypeToString(eType),
1857                                                                 sDevVAddrStart,
1858                                                                 sDevVAddrEnd,
1859                                                                 ui64TimeNs);
1860
1861                         pfnOSStatsPrintf(pvFilePtr, "%s\n", szBuffer);
1862                 }
1863         }
1864
1865         pfnOSStatsPrintf(pvFilePtr, "\nTimestamp reference: %013llu\n", ui64StartTime);
1866 }
1867
1868 static void DevicememHistoryPrintAllWrapper(void *pvFilePtr, void *pvData, OS_STATS_PRINTF_FUNC* pfnOSStatsPrintf)
1869 {
1870         PVR_UNREFERENCED_PARAMETER(pvData);
1871         DevicememHistoryLock();
1872         DevicememHistoryPrintAll(pvFilePtr, pfnOSStatsPrintf);
1873         DevicememHistoryUnlock();
1874 }
1875
1876 static PVRSRV_ERROR CreateRecords(void)
1877 {
1878         gsDevicememHistoryData.sRecords.pasAllocations =
1879                         OSAllocMem(sizeof(RECORD_ALLOCATION) * ALLOCATION_LIST_NUM_ENTRIES);
1880
1881         if(gsDevicememHistoryData.sRecords.pasAllocations == NULL)
1882         {
1883                 return PVRSRV_ERROR_OUT_OF_MEMORY;
1884         }
1885
1886         gsDevicememHistoryData.sRecords.pasCircularBuffer =
1887                         OSAllocMem(sizeof(COMMAND_WRAPPER) * CIRCULAR_BUFFER_NUM_COMMANDS);
1888
1889         if(gsDevicememHistoryData.sRecords.pasCircularBuffer == NULL)
1890         {
1891                 OSFreeMem(gsDevicememHistoryData.sRecords.pasAllocations);
1892                 return PVRSRV_ERROR_OUT_OF_MEMORY;
1893         }
1894
1895         return PVRSRV_OK;
1896 }
1897
1898 static void DestroyRecords(void)
1899 {
1900         OSFreeMem(gsDevicememHistoryData.sRecords.pasCircularBuffer);
1901         OSFreeMem(gsDevicememHistoryData.sRecords.pasAllocations);
1902 }
1903
1904 static void InitialiseRecords(void)
1905 {
1906         IMG_UINT32 i;
1907
1908         /* initialise the allocations list */
1909
1910         gsDevicememHistoryData.sRecords.pasAllocations[0].ui32Prev = ALLOCATION_LIST_NUM_ENTRIES - 1;
1911         gsDevicememHistoryData.sRecords.pasAllocations[0].ui32Next = 1;
1912
1913         for(i = 1; i < ALLOCATION_LIST_NUM_ENTRIES; i++)
1914         {
1915                 gsDevicememHistoryData.sRecords.pasAllocations[i].ui32Prev = i - 1;
1916                 gsDevicememHistoryData.sRecords.pasAllocations[i].ui32Next = i + 1;
1917         }
1918
1919         gsDevicememHistoryData.sRecords.pasAllocations[ALLOCATION_LIST_NUM_ENTRIES - 1].ui32Next = 0;
1920
1921         gsDevicememHistoryData.sRecords.ui32AllocationsListHead = 0;
1922
1923         /* initialise the circular buffer with zeros so every command
1924          * is initialised as a command of type COMMAND_TYPE_NONE
1925          */
1926         OSCachedMemSet(gsDevicememHistoryData.sRecords.pasCircularBuffer,
1927                                                                 COMMAND_TYPE_NONE,
1928                         sizeof(gsDevicememHistoryData.sRecords.pasCircularBuffer[0]) * CIRCULAR_BUFFER_NUM_COMMANDS);
1929 }
1930
1931 PVRSRV_ERROR DevicememHistoryInitKM(void)
1932 {
1933         PVRSRV_ERROR eError;
1934
1935         eError = OSLockCreate(&gsDevicememHistoryData.hLock, LOCK_TYPE_PASSIVE);
1936
1937         if(eError != PVRSRV_OK)
1938         {
1939                 PVR_DPF((PVR_DBG_ERROR, "DevicememHistoryInitKM: Failed to create lock"));
1940                 goto err_lock;
1941         }
1942
1943         eError = CreateRecords();
1944
1945         if(eError != PVRSRV_OK)
1946         {
1947                 PVR_DPF((PVR_DBG_ERROR, "DevicememHistoryInitKM: Failed to create records"));
1948                 goto err_allocations;
1949         }
1950
1951         InitialiseRecords();
1952
1953         gsDevicememHistoryData.pvStatsEntry = OSCreateStatisticEntry("devicemem_history",
1954                                                 NULL,
1955                                                 DevicememHistoryPrintAllWrapper,
1956                                                 NULL,
1957                                                 NULL,
1958                                                 NULL);
1959
1960         return PVRSRV_OK;
1961
1962 err_allocations:
1963         OSLockDestroy(gsDevicememHistoryData.hLock);
1964 err_lock:
1965         return eError;
1966 }
1967
1968 void DevicememHistoryDeInitKM(void)
1969 {
1970         if(gsDevicememHistoryData.pvStatsEntry != NULL)
1971         {
1972                 OSRemoveStatisticEntry(gsDevicememHistoryData.pvStatsEntry);
1973         }
1974
1975         DestroyRecords();
1976
1977         OSLockDestroy(gsDevicememHistoryData.hLock);
1978 }
1979
1980 PVRSRV_ERROR DevicememHistoryMapKM(IMG_DEV_VIRTADDR sDevVAddr, size_t uiSize, const char szString[DEVICEMEM_HISTORY_TEXT_BUFSZ])
1981 {
1982         IMG_UINT32 ui32AllocationIndex = DEVICEMEM_HISTORY_ALLOC_INDEX_NONE;
1983         IMG_UINT32 ui32Log2PageSize;
1984         IMG_UINT32 ui32StartPage;
1985         IMG_UINT32 ui32NumPages;
1986
1987         /* assume 4K page size */
1988         ui32Log2PageSize = 12;
1989
1990         ui32StartPage = 0;
1991         ui32NumPages = (uiSize + 4095) / 4096;
1992
1993         return DevicememHistoryMapVRangeKM(sDevVAddr,
1994                                                                 ui32StartPage,
1995                                                                 ui32NumPages,
1996                                                                 uiSize,
1997                                                                 szString,
1998                                                                 ui32Log2PageSize,
1999                                                                 ui32AllocationIndex,
2000                                                                 &ui32AllocationIndex);
2001 }
2002
2003 PVRSRV_ERROR DevicememHistoryUnmapKM(IMG_DEV_VIRTADDR sDevVAddr, size_t uiSize, const char szString[DEVICEMEM_HISTORY_TEXT_BUFSZ])
2004 {
2005         IMG_UINT32 ui32AllocationIndex = DEVICEMEM_HISTORY_ALLOC_INDEX_NONE;
2006         IMG_UINT32 ui32Log2PageSize;
2007         IMG_UINT32 ui32StartPage;
2008         IMG_UINT32 ui32NumPages;
2009
2010         /* assume 4K page size */
2011         ui32Log2PageSize = 12;
2012
2013         ui32StartPage = 0;
2014         ui32NumPages = (uiSize + 4095) / 4096;
2015
2016         return DevicememHistoryUnmapVRangeKM(sDevVAddr,
2017                                                                 ui32StartPage,
2018                                                                 ui32NumPages,
2019                                                                 uiSize,
2020                                                                 szString,
2021                                                                 ui32Log2PageSize,
2022                                                                 ui32AllocationIndex,
2023                                                                 &ui32AllocationIndex);
2024 }