drivers/staging/imgtec/rogue/rgxta3d.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          RGX TA/3D routines
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    RGX TA/3D routines
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43 /* for the offsetof macro */
  44 #include <stddef.h>
  45
  46 #include "pdump_km.h"
  47 #include "pvr_debug.h"
  48 #include "rgxutils.h"
  49 #include "rgxfwutils.h"
  50 #include "rgxta3d.h"
  51 #include "rgxmem.h"
  52 #include "allocmem.h"
  53 #include "devicemem.h"
  54 #include "devicemem_pdump.h"
  55 #include "ri_server.h"
  56 #include "osfunc.h"
  57 #include "pvrsrv.h"
  58 #include "rgx_memallocflags.h"
  59 #include "rgxccb.h"
  60 #include "rgxhwperf.h"
  61 #include "rgxtimerquery.h"
  62 #include "htbuffer.h"
  63
  64 #include "rgxdefs_km.h"
  65 #include "rgx_fwif_km.h"
  66 #include "physmem.h"
  67 #include "sync_server.h"
  68 #include "sync_internal.h"
  69 #include "sync.h"
  70 #include "process_stats.h"
  71
  72 #if defined(SUPPORT_BUFFER_SYNC)
  73 #include "pvr_buffer_sync.h"
  74 #endif
  75
  76 #if defined(SUPPORT_NATIVE_FENCE_SYNC)
  77 #include "pvr_sync.h"
  78 #endif
  79
  80 #if defined(SUPPORT_PDVFS)
  81 #include "rgxpdvfs.h"
  82 #endif
  83
  84 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
  85 #include "hash.h"
  86 #include "rgxworkest.h"
  87
  88 #define HASH_CLEAN_LIMIT 6
  89 #endif
  90
  91 typedef struct _DEVMEM_REF_LOOKUP_
  92 {
  93         IMG_UINT32 ui32ZSBufferID;
  94         RGX_ZSBUFFER_DATA *psZSBuffer;
  95 } DEVMEM_REF_LOOKUP;
  96
  97 typedef struct _DEVMEM_FREELIST_LOOKUP_
  98 {
  99         IMG_UINT32 ui32FreeListID;
 100         RGX_FREELIST *psFreeList;
 101 } DEVMEM_FREELIST_LOOKUP;
 102
 103 typedef struct {
 104         DEVMEM_MEMDESC                          *psContextStateMemDesc;
 105         RGX_SERVER_COMMON_CONTEXT       *psServerCommonContext;
 106         IMG_UINT32                                      ui32Priority;
 107 } RGX_SERVER_RC_TA_DATA;
 108
 109 typedef struct {
 110         DEVMEM_MEMDESC                          *psContextStateMemDesc;
 111         RGX_SERVER_COMMON_CONTEXT       *psServerCommonContext;
 112         IMG_UINT32                                      ui32Priority;
 113 } RGX_SERVER_RC_3D_DATA;
 114
 115 struct _RGX_SERVER_RENDER_CONTEXT_ {
 116         PVRSRV_DEVICE_NODE                      *psDeviceNode;
 117         DEVMEM_MEMDESC                          *psFWRenderContextMemDesc;
 118         DEVMEM_MEMDESC                          *psFWFrameworkMemDesc;
 119         RGX_SERVER_RC_TA_DATA           sTAData;
 120         RGX_SERVER_RC_3D_DATA           s3DData;
 121         IMG_UINT32                                      ui32CleanupStatus;
 122 #define RC_CLEANUP_TA_COMPLETE          (1 << 0)
 123 #define RC_CLEANUP_3D_COMPLETE          (1 << 1)
 124         PVRSRV_CLIENT_SYNC_PRIM         *psCleanupSync;
 125         DLLIST_NODE                                     sListNode;
 126         SYNC_ADDR_LIST                          sSyncAddrListTAFence;
 127         SYNC_ADDR_LIST                          sSyncAddrListTAUpdate;
 128         SYNC_ADDR_LIST                          sSyncAddrList3DFence;
 129         SYNC_ADDR_LIST                          sSyncAddrList3DUpdate;
 130         ATOMIC_T                                        hJobId;
 131 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
 132         WORKEST_HOST_DATA                       sWorkEstData;
 133 #endif
 134 };
 135
 136
 137 #if ! defined(NO_HARDWARE)
 138 static
 139 #ifdef __GNUC__
 140         __attribute__((noreturn))
 141 #endif
 142 void sleep_for_ever(void)
 143 {
 144 #if defined(__KLOCWORK__) // klocworks would report an infinite loop because of while(1).
 145         PVR_ASSERT(0);
 146 #else
 147         while(1)
 148         {
 149                 OSSleepms(~0); // sleep the maximum amount of time possible
 150         }
 151 #endif
 152 }
 153 #endif
 154
 155
 156 /*
 157         Static functions used by render context code
 158 */
 159
 160 static
 161 PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
 162                                                            PVRSRV_DEVICE_NODE *psDeviceNode,
 163                                                            PVRSRV_CLIENT_SYNC_PRIM *psCleanupSync)
 164 {
 165         PVRSRV_ERROR eError;
 166
 167         /* Check if the FW has finished with this resource ... */
 168         eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
 169                                                                                           psTAData->psServerCommonContext,
 170                                                                                           psCleanupSync,
 171                                                                                           RGXFWIF_DM_TA,
 172                                                                                           PDUMP_FLAGS_NONE);
 173         if (eError == PVRSRV_ERROR_RETRY)
 174         {
 175                 return eError;
 176         }
 177         else if (eError != PVRSRV_OK)
 178         {
 179                 PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
 180                                 __FUNCTION__,
 181                                 PVRSRVGetErrorStringKM(eError)));
 182                 return eError;
 183         }
 184
 185         /* ... it has so we can free it's resources */
 186 #if defined(DEBUG)
 187         /* Log the number of TA context stores which occurred */
 188         {
 189                 RGXFWIF_TACTX_STATE     *psFWTAState;
 190
 191                 eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
 192                                                                                   (void**)&psFWTAState);
 193                 if (eError != PVRSRV_OK)
 194                 {
 195                         PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware render context state (%u)",
 196                                         __FUNCTION__, eError));
 197                 }
 198                 else
 199                 {
 200                         /* Release the CPU virt addr */
 201                         DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
 202                 }
 203         }
 204 #endif
 205         FWCommonContextFree(psTAData->psServerCommonContext);
 206         DevmemFwFree(psDeviceNode->pvDevice, psTAData->psContextStateMemDesc);
 207         psTAData->psServerCommonContext = NULL;
 208         return PVRSRV_OK;
 209 }
 210
 211 static
 212 PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
 213                                                            PVRSRV_DEVICE_NODE *psDeviceNode,
 214                                                            PVRSRV_CLIENT_SYNC_PRIM *psCleanupSync)
 215 {
 216         PVRSRV_ERROR eError;
 217
 218         /* Check if the FW has finished with this resource ... */
 219         eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
 220                                                                                           ps3DData->psServerCommonContext,
 221                                                                                           psCleanupSync,
 222                                                                                           RGXFWIF_DM_3D,
 223                                                                                           PDUMP_FLAGS_NONE);
 224         if (eError == PVRSRV_ERROR_RETRY)
 225         {
 226                 return eError;
 227         }
 228         else if (eError != PVRSRV_OK)
 229         {
 230                 PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
 231                                  __FUNCTION__,
 232                                  PVRSRVGetErrorStringKM(eError)));
 233                 return eError;
 234         }
 235
 236         /* ... it has so we can free it's resources */
 237 #if defined(DEBUG)
 238         /* Log the number of 3D context stores which occurred */
 239         {
 240                 RGXFWIF_3DCTX_STATE     *psFW3DState;
 241
 242                 eError = DevmemAcquireCpuVirtAddr(ps3DData->psContextStateMemDesc,
 243                                                                                   (void**)&psFW3DState);
 244                 if (eError != PVRSRV_OK)
 245                 {
 246                         PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware render context state (%u)",
 247                                         __FUNCTION__, eError));
 248                 }
 249                 else
 250                 {
 251                         /* Release the CPU virt addr */
 252                         DevmemReleaseCpuVirtAddr(ps3DData->psContextStateMemDesc);
 253                 }
 254         }
 255 #endif
 256
 257         FWCommonContextFree(ps3DData->psServerCommonContext);
 258         DevmemFwFree(psDeviceNode->pvDevice, ps3DData->psContextStateMemDesc);
 259         ps3DData->psServerCommonContext = NULL;
 260         return PVRSRV_OK;
 261 }
 262
 263 static void _RGXDumpPMRPageList(DLLIST_NODE *psNode)
 264 {
 265         RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
 266         PVRSRV_ERROR                    eError;
 267
 268         eError = PMRDumpPageList(psPMRNode->psPMR,
 269                                                         RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
 270         if (eError != PVRSRV_OK)
 271         {
 272                 PVR_DPF((PVR_DBG_ERROR,"Error (%u) printing pmr %p", eError, psPMRNode->psPMR));
 273         }
 274 }
 275
 276 IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
 277 {
 278         DLLIST_NODE *psNode, *psNext;
 279
 280         PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016llx",
 281                                 psFreeList->sFreeListFWDevVAddr.ui32Addr,
 282                                 psFreeList->ui32FreelistID,
 283                                 psFreeList->ui64FreelistChecksum));
 284
 285         /* Dump Init FreeList page list */
 286         PVR_LOG(("  Initial Memory block"));
 287         dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
 288         {
 289                 _RGXDumpPMRPageList(psNode);
 290         }
 291
 292         /* Dump Grow FreeList page list */
 293         PVR_LOG(("  Grow Memory blocks"));
 294         dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
 295         {
 296                 _RGXDumpPMRPageList(psNode);
 297         }
 298
 299         return IMG_TRUE;
 300 }
 301
 302 static PVRSRV_ERROR _UpdateFwFreelistSize(RGX_FREELIST *psFreeList,
 303                                                                                 IMG_BOOL bGrow,
 304                                                                                 IMG_UINT32 ui32DeltaSize)
 305 {
 306         PVRSRV_ERROR                    eError = PVRSRV_OK;
 307         RGXFWIF_KCCB_CMD                sGPCCBCmd;
 308
 309         sGPCCBCmd.eCmdType = (bGrow) ? RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE : RGXFWIF_KCCB_CMD_FREELIST_SHRINK_UPDATE;
 310         sGPCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
 311         sGPCCBCmd.uCmdData.sFreeListGSData.ui32DeltaSize = ui32DeltaSize;
 312         sGPCCBCmd.uCmdData.sFreeListGSData.ui32NewSize = psFreeList->ui32CurrentFLPages;
 313
 314         PVR_DPF((PVR_DBG_MESSAGE, "Send FW update: freelist [FWAddr=0x%08x] has 0x%08x pages",
 315                                                                 psFreeList->sFreeListFWDevVAddr.ui32Addr,
 316                                                                 psFreeList->ui32CurrentFLPages));
 317
 318         /* Submit command to the firmware.  */
 319         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
 320         {
 321                 eError = RGXScheduleCommand(psFreeList->psDevInfo,
 322                                                                         RGXFWIF_DM_GP,
 323                                                                         &sGPCCBCmd,
 324                                                                         sizeof(sGPCCBCmd),
 325                                                                         0,
 326                                                                         PDUMP_FLAGS_CONTINUOUS);
 327                 if (eError != PVRSRV_ERROR_RETRY)
 328                 {
 329                         break;
 330                 }
 331                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
 332         } END_LOOP_UNTIL_TIMEOUT();
 333
 334         if (eError != PVRSRV_OK)
 335         {
 336                 PVR_DPF((PVR_DBG_ERROR, "_UpdateFwFreelistSize: failed to update FW freelist size. (error = %u)", eError));
 337                 return eError;
 338         }
 339
 340         return eError;
 341 }
 342
 343 static void _CheckFreelist(RGX_FREELIST *psFreeList,
 344                                                    IMG_UINT32 ui32NumOfPagesToCheck,
 345                                                    IMG_UINT64 ui64ExpectedCheckSum,
 346                                                    IMG_UINT64 *pui64CalculatedCheckSum)
 347 {
 348 #if defined(NO_HARDWARE)
 349         /* No checksum needed as we have all information in the pdumps */
 350         PVR_UNREFERENCED_PARAMETER(psFreeList);
 351         PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
 352         PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
 353         *pui64CalculatedCheckSum = 0;
 354 #else
 355         PVRSRV_ERROR eError;
 356         size_t uiNumBytes;
 357     IMG_UINT8* pui8Buffer;
 358     IMG_UINT32* pui32Buffer;
 359     IMG_UINT32 ui32CheckSumAdd = 0;
 360     IMG_UINT32 ui32CheckSumXor = 0;
 361     IMG_UINT32 ui32Entry;
 362     IMG_UINT32 ui32Entry2;
 363     IMG_BOOL  bFreelistBad = IMG_FALSE;
 364
 365         *pui64CalculatedCheckSum = 0;
 366
 367         /* Allocate Buffer of the size of the freelist */
 368         pui8Buffer = OSAllocMem(psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32));
 369     if (pui8Buffer == NULL)
 370     {
 371                 PVR_LOG(("_CheckFreelist: Failed to allocate buffer to check freelist %p!", psFreeList));
 372                 sleep_for_ever();
 373                 //PVR_ASSERT(0);
 374         return;
 375     }
 376
 377     /* Copy freelist content into Buffer */
 378     eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
 379                                 psFreeList->uiFreeListPMROffset + (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32),
 380                                 pui8Buffer,
 381                                 psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32),
 382                         &uiNumBytes);
 383     if (eError != PVRSRV_OK)
 384     {
 385                 OSFreeMem(pui8Buffer);
 386                 PVR_LOG(("_CheckFreelist: Failed to get freelist data for freelist %p!", psFreeList));
 387                 sleep_for_ever();
 388                 //PVR_ASSERT(0);
 389         return;
 390     }
 391
 392     PVR_ASSERT(uiNumBytes == psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32));
 393     PVR_ASSERT(ui32NumOfPagesToCheck <= psFreeList->ui32CurrentFLPages);
 394
 395     /* Generate checksum */
 396     pui32Buffer = (IMG_UINT32 *)pui8Buffer;
 397     for(ui32Entry = 0; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
 398     {
 399         ui32CheckSumAdd += pui32Buffer[ui32Entry];
 400         ui32CheckSumXor ^= pui32Buffer[ui32Entry];
 401
 402         /* Check for double entries */
 403         for (ui32Entry2 = 0; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
 404         {
 405                         if ((ui32Entry != ui32Entry2) &&
 406                                 (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2]))
 407                         {
 408                                 PVR_LOG(("_CheckFreelist: Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
 409                                                                                         psFreeList->sFreeListFWDevVAddr.ui32Addr,
 410                                                                                         pui32Buffer[ui32Entry2],
 411                                                                                         ui32Entry,
 412                                                                                         ui32Entry2,
 413                                                                                         psFreeList->ui32CurrentFLPages));
 414                                 bFreelistBad = IMG_TRUE;
 415                         }
 416         }
 417     }
 418
 419     OSFreeMem(pui8Buffer);
 420
 421         /* Check the calculated checksum against the expected checksum... */
 422         *pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
 423
 424         if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
 425         {
 426                 PVR_LOG(("_CheckFreelist: Checksum mismatch for freelist %p!  Expected 0x%016llx calculated 0x%016llx",
 427                         psFreeList, ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
 428                 bFreelistBad = IMG_TRUE;
 429         }
 430
 431     if (bFreelistBad)
 432     {
 433                 PVR_LOG(("_CheckFreelist: Sleeping for ever!"));
 434                 sleep_for_ever();
 435 //              PVR_ASSERT(!bFreelistBad);
 436         }
 437 #endif
 438 }
 439
 440 PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
 441                                                         IMG_UINT32 ui32NumPages,
 442                                                         PDLLIST_NODE pListHeader)
 443 {
 444         RGX_PMR_NODE    *psPMRNode;
 445         IMG_DEVMEM_SIZE_T uiSize;
 446         IMG_UINT32  ui32MappingTable = 0;
 447         IMG_DEVMEM_OFFSET_T uiOffset;
 448         IMG_DEVMEM_SIZE_T uiLength;
 449         IMG_DEVMEM_SIZE_T uistartPage;
 450         PVRSRV_ERROR eError;
 451         const IMG_CHAR * pszAllocName = "Free List";
 452
 453         /* Are we allowed to grow ? */
 454         if ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) < ui32NumPages)
 455         {
 456                 PVR_DPF((PVR_DBG_WARNING,"Freelist [0x%p]: grow by %u pages denied. Max PB size reached (current pages %u/%u)",
 457                                 psFreeList,
 458                                 ui32NumPages,
 459                                 psFreeList->ui32CurrentFLPages,
 460                                 psFreeList->ui32MaxFLPages));
 461                 return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
 462         }
 463
 464         /* Allocate kernel memory block structure */
 465         psPMRNode = OSAllocMem(sizeof(*psPMRNode));
 466         if (psPMRNode == NULL)
 467         {
 468                 PVR_DPF((PVR_DBG_ERROR, "RGXGrowFreeList: failed to allocate host data structure"));
 469                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 470                 goto ErrorAllocHost;
 471         }
 472
 473         /*
 474          * Lock protects simultaneous manipulation of:
 475          * - the memory block list
 476          * - the freelist's ui32CurrentFLPages
 477          */
 478         OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
 479
 480
 481         psPMRNode->ui32NumPages = ui32NumPages;
 482         psPMRNode->psFreeList = psFreeList;
 483
 484         /* Allocate Memory Block */
 485         PDUMPCOMMENT("Allocate PB Block (Pages %08X)", ui32NumPages);
 486         uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
 487         eError = PhysmemNewRamBackedPMR(NULL,
 488                                         psFreeList->psDevInfo->psDeviceNode,
 489                                                                         uiSize,
 490                                                                         uiSize,
 491                                                                         1,
 492                                                                         1,
 493                                                                         &ui32MappingTable,
 494                                                                         RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
 495                                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE,
 496                                                                         OSStringLength(pszAllocName) + 1,
 497                                                                         pszAllocName,
 498                                                                         &psPMRNode->psPMR);
 499         if(eError != PVRSRV_OK)
 500         {
 501                 PVR_DPF((PVR_DBG_ERROR,
 502                                  "RGXGrowFreeList: Failed to allocate PB block of size: 0x%016llX",
 503                                  (IMG_UINT64)uiSize));
 504                 goto ErrorBlockAlloc;
 505         }
 506
 507         /* Zeroing physical pages pointed by the PMR */
 508         if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
 509         {
 510                 eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
 511                 if (eError != PVRSRV_OK)
 512                 {
 513                         PVR_DPF((PVR_DBG_ERROR,"RGXGrowFreeList: Failed to zero PMR %p of freelist %p with Error %d",
 514                                                                         psPMRNode->psPMR,
 515                                                                         psFreeList,
 516                                                                         eError));
 517                         PVR_ASSERT(0);
 518                 }
 519         }
 520
 521         uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
 522         uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
 523         uiOffset = psFreeList->uiFreeListPMROffset + (uistartPage * sizeof(IMG_UINT32));
 524
 525 #if defined(PVR_RI_DEBUG)
 526
 527         eError = RIWritePMREntryKM(psPMRNode->psPMR,
 528                                    OSStringNLength(pszAllocName, RI_MAX_TEXT_LEN),
 529                                    pszAllocName,
 530                                    uiSize);
 531         if( eError != PVRSRV_OK)
 532         {
 533                 PVR_DPF((PVR_DBG_ERROR,
 534                                 "%s: call to RIWritePMREntryKM failed (eError=%d)",
 535                                 __func__,
 536                                 eError));
 537         }
 538
 539          /* Attach RI information */
 540         eError = RIWriteMEMDESCEntryKM(psPMRNode->psPMR,
 541                                        OSStringNLength(pszAllocName, RI_MAX_TEXT_LEN),
 542                                        pszAllocName,
 543                                        0,
 544                                        uiSize,
 545                                        uiSize,
 546                                        IMG_FALSE,
 547                                        IMG_FALSE,
 548                                        &psPMRNode->hRIHandle);
 549         if( eError != PVRSRV_OK)
 550         {
 551                 PVR_DPF((PVR_DBG_ERROR,
 552                                 "%s: call to RIWriteMEMDESCEntryKM failed (eError=%d)",
 553                                 __func__,
 554                                 eError));
 555         }
 556
 557 #endif /* if defined(PVR_RI_DEBUG) */
 558
 559         /* write Freelist with Memory Block physical addresses */
 560         eError = PMRWritePMPageList(
 561                                                 /* Target PMR, offset, and length */
 562                                                 psFreeList->psFreeListPMR,
 563                                                 uiOffset,
 564                                                 uiLength,
 565                                                 /* Referenced PMR, and "page" granularity */
 566                                                 psPMRNode->psPMR,
 567                                                 RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
 568                                                 &psPMRNode->psPageList);
 569         if (eError != PVRSRV_OK)
 570         {
 571                 PVR_DPF((PVR_DBG_ERROR,
 572                                  "RGXGrowFreeList: Failed to write pages of Node %p",
 573                                  psPMRNode));
 574                 goto ErrorPopulateFreelist;
 575         }
 576
 577         /* We add It must be added to the tail, otherwise the freelist population won't work */
 578         dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
 579
 580         /* Update number of available pages */
 581         psFreeList->ui32CurrentFLPages += ui32NumPages;
 582
 583         /* Update statistics */
 584         if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
 585         {
 586                 psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
 587         }
 588
 589         if (psFreeList->bCheckFreelist)
 590         {
 591                 /* We can only do a freelist check if the list is full (e.g. at initial creation time) */
 592                 if (psFreeList->ui32CurrentFLPages == ui32NumPages)
 593                 {
 594                         IMG_UINT64  ui64Dummy;
 595                         _CheckFreelist(psFreeList, ui32NumPages, psFreeList->ui64FreelistChecksum, &ui64Dummy);
 596                 }
 597         }
 598
 599         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
 600
 601         PVR_DPF((PVR_DBG_MESSAGE,"Freelist [%p]: grow by %u pages (current pages %u/%u)",
 602                         psFreeList,
 603                         ui32NumPages,
 604                         psFreeList->ui32CurrentFLPages,
 605                         psFreeList->ui32MaxFLPages));
 606
 607         return PVRSRV_OK;
 608
 609         /* Error handling */
 610 ErrorPopulateFreelist:
 611         PMRUnrefPMR(psPMRNode->psPMR);
 612
 613 ErrorBlockAlloc:
 614         OSFreeMem(psPMRNode);
 615         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
 616
 617 ErrorAllocHost:
 618         PVR_ASSERT(eError != PVRSRV_OK);
 619         return eError;
 620
 621 }
 622
 623 static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
 624                                                                                 RGX_FREELIST *psFreeList)
 625 {
 626         DLLIST_NODE *psNode;
 627         RGX_PMR_NODE *psPMRNode;
 628         PVRSRV_ERROR eError = PVRSRV_OK;
 629         IMG_UINT32 ui32OldValue;
 630
 631         /*
 632          * Lock protects simultaneous manipulation of:
 633          * - the memory block list
 634          * - the freelist's ui32CurrentFLPages value
 635          */
 636         PVR_ASSERT(pListHeader);
 637         PVR_ASSERT(psFreeList);
 638         PVR_ASSERT(psFreeList->psDevInfo);
 639         PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
 640
 641         OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
 642
 643         /* Get node from head of list and remove it */
 644         psNode = dllist_get_next_node(pListHeader);
 645         if (psNode)
 646         {
 647                 dllist_remove_node(psNode);
 648
 649                 psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
 650                 PVR_ASSERT(psPMRNode);
 651                 PVR_ASSERT(psPMRNode->psPMR);
 652                 PVR_ASSERT(psPMRNode->psFreeList);
 653
 654                 /* remove block from freelist list */
 655
 656                 /* Unwrite Freelist with Memory Block physical addresses */
 657                 eError = PMRUnwritePMPageList(psPMRNode->psPageList);
 658                 if (eError != PVRSRV_OK)
 659                 {
 660                         PVR_DPF((PVR_DBG_ERROR,
 661                                          "RGXRemoveBlockFromFreeListKM: Failed to unwrite pages of Node %p",
 662                                          psPMRNode));
 663                         PVR_ASSERT(IMG_FALSE);
 664                 }
 665
 666 #if defined(PVR_RI_DEBUG)
 667
 668                 if (psPMRNode->hRIHandle)
 669                 {
 670                     PVRSRV_ERROR eError;
 671
 672                     eError = RIDeleteMEMDESCEntryKM(psPMRNode->hRIHandle);
 673                         if( eError != PVRSRV_OK)
 674                         {
 675                                 PVR_DPF((PVR_DBG_ERROR, "%s: call to RIDeleteMEMDESCEntryKM failed (eError=%d)", __func__, eError));
 676                         }
 677                 }
 678
 679 #endif  /* if defined(PVR_RI_DEBUG) */
 680
 681                 /* Free PMR (We should be the only one that holds a ref on the PMR) */
 682                 eError = PMRUnrefPMR(psPMRNode->psPMR);
 683                 if (eError != PVRSRV_OK)
 684                 {
 685                         PVR_DPF((PVR_DBG_ERROR,
 686                                          "RGXRemoveBlockFromFreeListKM: Failed to free PB block %p (error %u)",
 687                                          psPMRNode->psPMR,
 688                                          eError));
 689                         PVR_ASSERT(IMG_FALSE);
 690                 }
 691
 692                 /* update available pages in freelist */
 693                 ui32OldValue = psFreeList->ui32CurrentFLPages;
 694                 psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages;
 695
 696                 /* check underflow */
 697                 PVR_ASSERT(ui32OldValue > psFreeList->ui32CurrentFLPages);
 698
 699                 PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
 700                                                                 psFreeList,
 701                                                                 psPMRNode->ui32NumPages,
 702                                                                 psFreeList->ui32CurrentFLPages,
 703                                                                 psFreeList->ui32MaxFLPages));
 704
 705                 OSFreeMem(psPMRNode);
 706         }
 707         else
 708         {
 709                 PVR_DPF((PVR_DBG_WARNING,"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
 710                                                                 psFreeList,
 711                                                                 psFreeList->ui32InitFLPages));
 712                 eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
 713         }
 714
 715         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
 716
 717         return eError;
 718 }
 719
 720 static RGX_FREELIST *FindFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
 721 {
 722         DLLIST_NODE *psNode, *psNext;
 723         RGX_FREELIST *psFreeList = NULL;
 724
 725         OSLockAcquire(psDevInfo->hLockFreeList);
 726
 727         dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
 728         {
 729                 RGX_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
 730
 731                 if (psThisFreeList->ui32FreelistID == ui32FreelistID)
 732                 {
 733                         psFreeList = psThisFreeList;
 734                         break;
 735                 }
 736         }
 737
 738         OSLockRelease(psDevInfo->hLockFreeList);
 739         return psFreeList;
 740 }
 741
 742 void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
 743                                                    IMG_UINT32 ui32FreelistID)
 744 {
 745         RGX_FREELIST *psFreeList = NULL;
 746         RGXFWIF_KCCB_CMD s3DCCBCmd;
 747         IMG_UINT32 ui32GrowValue;
 748         PVRSRV_ERROR eError;
 749
 750         PVR_ASSERT(psDevInfo);
 751
 752         psFreeList = FindFreeList(psDevInfo, ui32FreelistID);
 753
 754         if (psFreeList)
 755         {
 756                 /* Try to grow the freelist */
 757                 eError = RGXGrowFreeList(psFreeList,
 758                                                                 psFreeList->ui32GrowFLPages,
 759                                                                 &psFreeList->sMemoryBlockHead);
 760                 if (eError == PVRSRV_OK)
 761                 {
 762                         /* Grow successful, return size of grow size */
 763                         ui32GrowValue = psFreeList->ui32GrowFLPages;
 764
 765                         psFreeList->ui32NumGrowReqByFW++;
 766
 767  #if defined(PVRSRV_ENABLE_PROCESS_STATS)
 768                         /* Update Stats */
 769                         PVRSRVStatsUpdateFreelistStats(0,
 770                                        1, /* Add 1 to the appropriate counter (Requests by FW) */
 771                                        psFreeList->ui32InitFLPages,
 772                                        psFreeList->ui32NumHighPages,
 773                                        psFreeList->ownerPid);
 774
 775  #endif
 776
 777                 }
 778                 else
 779                 {
 780                         /* Grow failed */
 781                         ui32GrowValue = 0;
 782                         PVR_DPF((PVR_DBG_ERROR,"Grow for FreeList %p failed (error %u)",
 783                                                                         psFreeList,
 784                                                                         eError));
 785                 }
 786
 787                 /* send feedback */
 788                 s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
 789                 s3DCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
 790                 s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaSize = ui32GrowValue;
 791                 s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewSize = psFreeList->ui32CurrentFLPages;
 792
 793                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
 794                 {
 795                         eError = RGXScheduleCommand(psDevInfo,
 796                                                                                 RGXFWIF_DM_3D,
 797                                                                                 &s3DCCBCmd,
 798                                                                                 sizeof(s3DCCBCmd),
 799                                                                                 0,
 800                                                                                 PDUMP_FLAGS_NONE);
 801                         if (eError != PVRSRV_ERROR_RETRY)
 802                         {
 803                                 break;
 804                         }
 805                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
 806                 } END_LOOP_UNTIL_TIMEOUT();
 807                 /* Kernel CCB should never fill up, as the FW is processing them right away  */
 808
 809                 PVR_ASSERT(eError == PVRSRV_OK);
 810         }
 811         else
 812         {
 813                 /* Should never happen */
 814                 PVR_DPF((PVR_DBG_ERROR,"FreeList Lookup for FreeList ID 0x%08x failed (Populate)", ui32FreelistID));
 815                 PVR_ASSERT(IMG_FALSE);
 816         }
 817 }
 818
 819 static void _RGXCheckFreeListReconstruction(PDLLIST_NODE psNode)
 820 {
 821
 822         PVRSRV_RGXDEV_INFO              *psDevInfo;
 823         RGX_FREELIST                    *psFreeList;
 824         RGX_PMR_NODE                    *psPMRNode;
 825         PVRSRV_ERROR                    eError;
 826         IMG_DEVMEM_OFFSET_T             uiOffset;
 827         IMG_DEVMEM_SIZE_T               uiLength;
 828         IMG_UINT32                              ui32StartPage;
 829
 830         psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
 831         psFreeList = psPMRNode->psFreeList;
 832         PVR_ASSERT(psFreeList);
 833         psDevInfo = psFreeList->psDevInfo;
 834         PVR_ASSERT(psDevInfo);
 835
 836         uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
 837         ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
 838         uiOffset = psFreeList->uiFreeListPMROffset + (ui32StartPage * sizeof(IMG_UINT32));
 839
 840         PMRUnwritePMPageList(psPMRNode->psPageList);
 841         psPMRNode->psPageList = NULL;
 842         eError = PMRWritePMPageList(
 843                                                 /* Target PMR, offset, and length */
 844                                                 psFreeList->psFreeListPMR,
 845                                                 uiOffset,
 846                                                 uiLength,
 847                                                 /* Referenced PMR, and "page" granularity */
 848                                                 psPMRNode->psPMR,
 849                                                 RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
 850                                                 &psPMRNode->psPageList);
 851
 852         if (eError != PVRSRV_OK)
 853         {
 854                 PVR_DPF((PVR_DBG_ERROR,"Error (%u) writing FL 0x%08x", eError, (IMG_UINT32)psFreeList->ui32FreelistID));
 855         }
 856
 857         /* Zeroing physical pages pointed by the reconstructed freelist */
 858         if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
 859         {
 860                 eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
 861                 if (eError != PVRSRV_OK)
 862                 {
 863                         PVR_DPF((PVR_DBG_ERROR,"_RGXCheckFreeListReconstruction: Failed to zero PMR %p of freelist %p with Error %d",
 864                                                                         psPMRNode->psPMR,
 865                                                                         psFreeList,
 866                                                                         eError));
 867                         PVR_ASSERT(0);
 868                 }
 869         }
 870
 871         psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
 872 }
 873
 874
 875 static PVRSRV_ERROR RGXReconstructFreeList(RGX_FREELIST *psFreeList)
 876 {
 877         DLLIST_NODE       *psNode, *psNext;
 878         RGXFWIF_FREELIST  *psFWFreeList;
 879         PVRSRV_ERROR      eError;
 880
 881         //PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: Reconstructing freelist %p (ID=%u)", psFreeList, psFreeList->ui32FreelistID));
 882
 883         /* Do the FreeList Reconstruction */
 884         psFreeList->ui32CurrentFLPages = 0;
 885
 886         /* Reconstructing Init FreeList pages */
 887         dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
 888         {
 889                 _RGXCheckFreeListReconstruction(psNode);
 890         }
 891
 892         /* Reconstructing Grow FreeList pages */
 893         dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
 894         {
 895                 _RGXCheckFreeListReconstruction(psNode);
 896         }
 897
 898         /* Reset the firmware freelist structure */
 899         eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
 900         if (eError != PVRSRV_OK)
 901         {
 902                 return eError;
 903         }
 904
 905         psFWFreeList->ui32CurrentStackTop       = psFWFreeList->ui32CurrentPages - 1;
 906         psFWFreeList->ui32AllocatedPageCount    = 0;
 907         psFWFreeList->ui32AllocatedMMUPageCount = 0;
 908         psFWFreeList->ui32HWRCounter++;
 909
 910         DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
 911
 912         /* Check the Freelist checksum if required (as the list is fully populated) */
 913         if (psFreeList->bCheckFreelist)
 914         {
 915                 IMG_UINT64  ui64CheckSum;
 916
 917                 _CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
 918         }
 919
 920         return eError;
 921 }
 922
 923
 924 void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
 925                                               IMG_UINT32 ui32FreelistsCount,
 926                                               IMG_UINT32 *paui32Freelists)
 927 {
 928         PVRSRV_ERROR      eError = PVRSRV_OK;
 929         DLLIST_NODE       *psNode, *psNext;
 930         IMG_UINT32        ui32Loop;
 931         RGXFWIF_KCCB_CMD  sTACCBCmd;
 932
 933         PVR_ASSERT(psDevInfo != NULL);
 934         PVR_ASSERT(ui32FreelistsCount <= (MAX_HW_TA3DCONTEXTS * RGXFW_MAX_FREELISTS));
 935
 936         //PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: %u freelist(s) requested for reconstruction", ui32FreelistsCount));
 937
 938         /*
 939          *  Initialise the response command (in case we don't find a freelist ID)...
 940          */
 941         sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
 942         sTACCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
 943
 944         for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
 945         {
 946                 sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] = paui32Freelists[ui32Loop] |
 947                                                                                              RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
 948         }
 949
 950         /*
 951          *  The list of freelists we have been given for reconstruction will
 952          *  consist of local and global freelists (maybe MMU as well). Any
 953          *  local freelists will have their global list specified as well.
 954          *  However there may be other local freelists not listed, which are
 955          *  going to have their global freelist reconstructed. Therefore we
 956          *  have to find those freelists as well meaning we will have to
 957          *  iterate the entire list of freelists to find which must be reconstructed.
 958          */
 959         OSLockAcquire(psDevInfo->hLockFreeList);
 960         dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
 961         {
 962                 RGX_FREELIST  *psFreeList  = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
 963                 IMG_BOOL      bReconstruct = IMG_FALSE;
 964
 965                 /*
 966                  *  Check if this freelist needs to be reconstructed (was it requested
 967                  *  or was its global freelist requested)...
 968                  */
 969                 for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
 970                 {
 971                         if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID  ||
 972                             paui32Freelists[ui32Loop] == psFreeList->ui32FreelistGlobalID)
 973                         {
 974                                 bReconstruct = IMG_TRUE;
 975                                 break;
 976                         }
 977                 }
 978
 979                 if (bReconstruct)
 980                 {
 981                         eError = RGXReconstructFreeList(psFreeList);
 982                         if (eError == PVRSRV_OK)
 983                         {
 984                                 for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
 985                                 {
 986                                         if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
 987                                         {
 988                                                 /* Reconstruction of this requested freelist was successful... */
 989                                                 sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &= ~RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
 990                                                 break;
 991                                         }
 992                                 }
 993                         }
 994                         else
 995                         {
 996                                 PVR_DPF((PVR_DBG_ERROR,"Reconstructing of FreeList %p failed (error %u)",
 997                                                  psFreeList,
 998                                                  eError));
 999                         }
1000                 }
1001         }
1002         OSLockRelease(psDevInfo->hLockFreeList);
1003
1004         /* Check that all freelists were found and reconstructed... */
1005         for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
1006         {
1007                 PVR_ASSERT((sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &
1008                             RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG) == 0);
1009         }
1010
1011         /* send feedback */
1012         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
1013         {
1014                 eError = RGXScheduleCommand(psDevInfo,
1015                                             RGXFWIF_DM_TA,
1016                                             &sTACCBCmd,
1017                                             sizeof(sTACCBCmd),
1018                                             0,
1019                                             PDUMP_FLAGS_NONE);
1020                 if (eError != PVRSRV_ERROR_RETRY)
1021                 {
1022                         break;
1023                 }
1024                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
1025         } END_LOOP_UNTIL_TIMEOUT();
1026
1027         /* Kernel CCB should never fill up, as the FW is processing them right away  */
1028         PVR_ASSERT(eError == PVRSRV_OK);
1029 }
1030
1031 /* Create HWRTDataSet */
1032 IMG_EXPORT
1033 PVRSRV_ERROR RGXCreateHWRTData(CONNECTION_DATA      *psConnection,
1034                                PVRSRV_DEVICE_NODE       *psDeviceNode,
1035                                                            IMG_UINT32                   psRenderTarget, /* FIXME this should not be IMG_UINT32 */
1036                                                            IMG_DEV_VIRTADDR             psPMMListDevVAddr,
1037                                                            IMG_DEV_VIRTADDR             psVFPPageTableAddr,
1038                                                            RGX_FREELIST                 *apsFreeLists[RGXFW_MAX_FREELISTS],
1039                                                            RGX_RTDATA_CLEANUP_DATA      **ppsCleanupData,
1040                                                            DEVMEM_MEMDESC               **ppsRTACtlMemDesc,
1041                                                            IMG_UINT32           ui32PPPScreen,
1042                                                            IMG_UINT32           ui32PPPGridOffset,
1043                                                            IMG_UINT64           ui64PPPMultiSampleCtl,
1044                                                            IMG_UINT32           ui32TPCStride,
1045                                                            IMG_DEV_VIRTADDR             sTailPtrsDevVAddr,
1046                                                            IMG_UINT32           ui32TPCSize,
1047                                                            IMG_UINT32           ui32TEScreen,
1048                                                            IMG_UINT32           ui32TEAA,
1049                                                            IMG_UINT32           ui32TEMTILE1,
1050                                                            IMG_UINT32           ui32TEMTILE2,
1051                                                            IMG_UINT32           ui32MTileStride,
1052                                                            IMG_UINT32                 ui32ISPMergeLowerX,
1053                                                            IMG_UINT32                 ui32ISPMergeLowerY,
1054                                                            IMG_UINT32                 ui32ISPMergeUpperX,
1055                                                            IMG_UINT32                 ui32ISPMergeUpperY,
1056                                                            IMG_UINT32                 ui32ISPMergeScaleX,
1057                                                            IMG_UINT32                 ui32ISPMergeScaleY,
1058                                                            IMG_UINT16                   ui16MaxRTs,
1059                                                            DEVMEM_MEMDESC               **ppsMemDesc,
1060                                                            IMG_UINT32                   *puiHWRTData)
1061 {
1062         PVRSRV_ERROR eError;
1063         PVRSRV_RGXDEV_INFO *psDevInfo;
1064         RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
1065         RGXFWIF_HWRTDATA *psHWRTData;
1066         RGXFWIF_RTA_CTL *psRTACtl;
1067         IMG_UINT32 ui32Loop;
1068         RGX_RTDATA_CLEANUP_DATA *psTmpCleanup;
1069
1070         PVR_UNREFERENCED_PARAMETER(psConnection);
1071
1072         /* Prepare cleanup struct */
1073         psTmpCleanup = OSAllocZMem(sizeof(*psTmpCleanup));
1074         if (psTmpCleanup == NULL)
1075         {
1076                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1077                 goto AllocError;
1078         }
1079
1080         *ppsCleanupData = psTmpCleanup;
1081
1082         /* Allocate cleanup sync */
1083         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
1084                                                    &psTmpCleanup->psCleanupSync,
1085                                                    "HWRTData cleanup");
1086         if (eError != PVRSRV_OK)
1087         {
1088                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate cleanup sync (0x%x)",
1089                                 eError));
1090                 goto SyncAlloc;
1091         }
1092
1093         psDevInfo = psDeviceNode->pvDevice;
1094
1095         /*
1096          * This FW RT-Data is only mapped into kernel for initialisation.
1097          * Otherwise this allocation is only used by the FW.
1098          * Therefore the GPU cache doesn't need coherency,
1099          * and write-combine is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
1100          */
1101         eError = DevmemFwAllocate(psDevInfo,
1102                                                         sizeof(RGXFWIF_HWRTDATA),
1103                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1104                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
1105                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1106                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1107                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1108                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1109                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1110                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1111                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1112                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1113                                                         "FwHWRTData",
1114                                                         ppsMemDesc);
1115         if (eError != PVRSRV_OK)
1116         {
1117                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateHWRTData: DevmemAllocate for RGX_FWIF_HWRTDATA failed"));
1118                 goto FWRTDataAllocateError;
1119         }
1120
1121         psTmpCleanup->psDeviceNode = psDeviceNode;
1122         psTmpCleanup->psFWHWRTDataMemDesc = *ppsMemDesc;
1123
1124         RGXSetFirmwareAddress(&pFirmwareAddr, *ppsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1125
1126         *puiHWRTData = pFirmwareAddr.ui32Addr;
1127
1128         eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, (void **)&psHWRTData);
1129         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
1130
1131         /* FIXME: MList is something that that PM writes physical addresses to,
1132          * so ideally its best allocated in kernel */
1133         psHWRTData->psPMMListDevVAddr = psPMMListDevVAddr;
1134         psHWRTData->psParentRenderTarget.ui32Addr = psRenderTarget;
1135         #if defined(SUPPORT_VFP)
1136         psHWRTData->sVFPPageTableAddr = psVFPPageTableAddr;
1137         #endif
1138
1139         psHWRTData->ui32PPPScreen         = ui32PPPScreen;
1140         psHWRTData->ui32PPPGridOffset     = ui32PPPGridOffset;
1141         psHWRTData->ui64PPPMultiSampleCtl = ui64PPPMultiSampleCtl;
1142         psHWRTData->ui32TPCStride         = ui32TPCStride;
1143         psHWRTData->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
1144         psHWRTData->ui32TPCSize           = ui32TPCSize;
1145         psHWRTData->ui32TEScreen          = ui32TEScreen;
1146         psHWRTData->ui32TEAA              = ui32TEAA;
1147         psHWRTData->ui32TEMTILE1          = ui32TEMTILE1;
1148         psHWRTData->ui32TEMTILE2          = ui32TEMTILE2;
1149         psHWRTData->ui32MTileStride       = ui32MTileStride;
1150         psHWRTData->ui32ISPMergeLowerX = ui32ISPMergeLowerX;
1151         psHWRTData->ui32ISPMergeLowerY = ui32ISPMergeLowerY;
1152         psHWRTData->ui32ISPMergeUpperX = ui32ISPMergeUpperX;
1153         psHWRTData->ui32ISPMergeUpperY = ui32ISPMergeUpperY;
1154         psHWRTData->ui32ISPMergeScaleX = ui32ISPMergeScaleX;
1155         psHWRTData->ui32ISPMergeScaleY = ui32ISPMergeScaleY;
1156
1157         OSLockAcquire(psDevInfo->hLockFreeList);
1158         for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
1159         {
1160                 psTmpCleanup->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
1161                 psTmpCleanup->apsFreeLists[ui32Loop]->ui32RefCount++;
1162                 psHWRTData->apsFreeLists[ui32Loop].ui32Addr = psTmpCleanup->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr;
1163                 /* invalid initial snapshot value, the snapshot is always taken during first kick
1164                  * and hence the value get replaced during the first kick anyway. So it's safe to set it 0.
1165                 */
1166                 psHWRTData->aui32FreeListHWRSnapshot[ui32Loop] = 0;
1167         }
1168         OSLockRelease(psDevInfo->hLockFreeList);
1169
1170         PDUMPCOMMENT("Allocate RGXFW RTA control");
1171         eError = DevmemFwAllocate(psDevInfo,
1172                                                                                 sizeof(RGXFWIF_RTA_CTL),
1173                                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1174                                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
1175                                                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1176                                                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1177                                                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1178                                                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
1179                                                                                 "FwRTAControl",
1180                                                                                 ppsRTACtlMemDesc);
1181
1182         if (eError != PVRSRV_OK)
1183         {
1184                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate RGX RTA control (%u)",
1185                                 eError));
1186                 goto FWRTAAllocateError;
1187         }
1188         psTmpCleanup->psRTACtlMemDesc = *ppsRTACtlMemDesc;
1189         RGXSetFirmwareAddress(&psHWRTData->psRTACtl,
1190                                                                    *ppsRTACtlMemDesc,
1191                                                                    0, RFW_FWADDR_FLAG_NONE);
1192
1193         eError = DevmemAcquireCpuVirtAddr(*ppsRTACtlMemDesc, (void **)&psRTACtl);
1194         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTACpuMapError);
1195         psRTACtl->ui32RenderTargetIndex = 0;
1196         psRTACtl->ui32ActiveRenderTargets = 0;
1197
1198         if (ui16MaxRTs > 1)
1199         {
1200                 /* Allocate memory for the checks */
1201                 PDUMPCOMMENT("Allocate memory for shadow render target cache");
1202                 eError = DevmemFwAllocate(psDevInfo,
1203                                                                 ui16MaxRTs * sizeof(IMG_UINT32),
1204                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1205                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
1206                                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1207                                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1208                                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1209                                                                 PVRSRV_MEMALLOCFLAG_UNCACHED|
1210                                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
1211                                                                 "FwShadowRTCache",
1212                                                                 &psTmpCleanup->psRTArrayMemDesc);
1213                 if (eError != PVRSRV_OK)
1214                 {
1215                         PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate %d bytes for render target array (%u)",
1216                                 ui16MaxRTs, eError));
1217                         goto FWAllocateRTArryError;
1218                 }
1219
1220                 RGXSetFirmwareAddress(&psRTACtl->sValidRenderTargets,
1221                                                                                 psTmpCleanup->psRTArrayMemDesc,
1222                                                                                 0, RFW_FWADDR_FLAG_NONE);
1223
1224                 /* Allocate memory for the checks */
1225                 PDUMPCOMMENT("Allocate memory for tracking renders accumulation");
1226                 eError = DevmemFwAllocate(psDevInfo,
1227                                                         ui16MaxRTs * sizeof(IMG_UINT32),
1228                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1229                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
1230                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1231                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1232                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1233                                                         PVRSRV_MEMALLOCFLAG_UNCACHED|
1234                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
1235                                                         "FwRendersAccumulation",
1236                                                         &psTmpCleanup->psRendersAccArrayMemDesc);
1237                 if (eError != PVRSRV_OK)
1238                 {
1239                         PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate %d bytes for render target array (%u) (renders accumulation)",
1240                                                   ui16MaxRTs, eError));
1241                         goto FWAllocateRTAccArryError;
1242                 }
1243
1244                 RGXSetFirmwareAddress(&psRTACtl->sNumRenders,
1245                                                 psTmpCleanup->psRendersAccArrayMemDesc,
1246                                                 0, RFW_FWADDR_FLAG_NONE);
1247                 psRTACtl->ui16MaxRTs = ui16MaxRTs;
1248         }
1249         else
1250         {
1251                 psRTACtl->sValidRenderTargets.ui32Addr = 0;
1252                 psRTACtl->sNumRenders.ui32Addr = 0;
1253                 psRTACtl->ui16MaxRTs = 1;
1254         }
1255
1256         PDUMPCOMMENT("Dump HWRTData 0x%08X", *puiHWRTData);
1257         DevmemPDumpLoadMem(*ppsMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
1258         PDUMPCOMMENT("Dump RTACtl");
1259         DevmemPDumpLoadMem(*ppsRTACtlMemDesc, 0, sizeof(*psRTACtl), PDUMP_FLAGS_CONTINUOUS);
1260
1261         DevmemReleaseCpuVirtAddr(*ppsMemDesc);
1262         DevmemReleaseCpuVirtAddr(*ppsRTACtlMemDesc);
1263         return PVRSRV_OK;
1264
1265 FWAllocateRTAccArryError:
1266         DevmemFwFree(psDevInfo, psTmpCleanup->psRTArrayMemDesc);
1267 FWAllocateRTArryError:
1268         DevmemReleaseCpuVirtAddr(*ppsRTACtlMemDesc);
1269 FWRTACpuMapError:
1270         RGXUnsetFirmwareAddress(*ppsRTACtlMemDesc);
1271         DevmemFwFree(psDevInfo, *ppsRTACtlMemDesc);
1272 FWRTAAllocateError:
1273         OSLockAcquire(psDevInfo->hLockFreeList);
1274         for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
1275         {
1276                 PVR_ASSERT(psTmpCleanup->apsFreeLists[ui32Loop]->ui32RefCount > 0);
1277                 psTmpCleanup->apsFreeLists[ui32Loop]->ui32RefCount--;
1278         }
1279         OSLockRelease(psDevInfo->hLockFreeList);
1280         DevmemReleaseCpuVirtAddr(*ppsMemDesc);
1281 FWRTDataCpuMapError:
1282         RGXUnsetFirmwareAddress(*ppsMemDesc);
1283         DevmemFwFree(psDevInfo, *ppsMemDesc);
1284 FWRTDataAllocateError:
1285         SyncPrimFree(psTmpCleanup->psCleanupSync);
1286 SyncAlloc:
1287         *ppsCleanupData = NULL;
1288         OSFreeMem(psTmpCleanup);
1289
1290 AllocError:
1291         return eError;
1292 }
1293
1294 /* Destroy HWRTDataSet */
1295 IMG_EXPORT
1296 PVRSRV_ERROR RGXDestroyHWRTData(RGX_RTDATA_CLEANUP_DATA *psCleanupData)
1297 {
1298         PVRSRV_RGXDEV_INFO *psDevInfo;
1299         PVRSRV_ERROR eError;
1300         PRGXFWIF_HWRTDATA psHWRTData;
1301         IMG_UINT32 ui32Loop;
1302
1303         PVR_ASSERT(psCleanupData);
1304
1305         RGXSetFirmwareAddress(&psHWRTData, psCleanupData->psFWHWRTDataMemDesc, 0, RFW_FWADDR_NOREF_FLAG);
1306
1307         /* Cleanup HWRTData in TA */
1308         eError = RGXFWRequestHWRTDataCleanUp(psCleanupData->psDeviceNode,
1309                                                                                  psHWRTData,
1310                                                                                  psCleanupData->psCleanupSync,
1311                                                                                  RGXFWIF_DM_TA);
1312         if (eError == PVRSRV_ERROR_RETRY)
1313         {
1314                 return eError;
1315         }
1316
1317         psDevInfo = psCleanupData->psDeviceNode->pvDevice;
1318
1319         /* Cleanup HWRTData in 3D */
1320         eError = RGXFWRequestHWRTDataCleanUp(psCleanupData->psDeviceNode,
1321                                                                                  psHWRTData,
1322                                                                                  psCleanupData->psCleanupSync,
1323                                                                                  RGXFWIF_DM_3D);
1324         if (eError == PVRSRV_ERROR_RETRY)
1325         {
1326                 return eError;
1327         }
1328
1329         /* If we got here then TA and 3D operations on this RTData have finished */
1330         if(psCleanupData->psRTACtlMemDesc)
1331         {
1332                 RGXUnsetFirmwareAddress(psCleanupData->psRTACtlMemDesc);
1333                 DevmemFwFree(psDevInfo, psCleanupData->psRTACtlMemDesc);
1334         }
1335
1336         RGXUnsetFirmwareAddress(psCleanupData->psFWHWRTDataMemDesc);
1337         DevmemFwFree(psDevInfo, psCleanupData->psFWHWRTDataMemDesc);
1338
1339         if(psCleanupData->psRTArrayMemDesc)
1340         {
1341                 RGXUnsetFirmwareAddress(psCleanupData->psRTArrayMemDesc);
1342                 DevmemFwFree(psDevInfo, psCleanupData->psRTArrayMemDesc);
1343         }
1344
1345         if(psCleanupData->psRendersAccArrayMemDesc)
1346         {
1347                 RGXUnsetFirmwareAddress(psCleanupData->psRendersAccArrayMemDesc);
1348                 DevmemFwFree(psDevInfo, psCleanupData->psRendersAccArrayMemDesc);
1349         }
1350
1351         SyncPrimFree(psCleanupData->psCleanupSync);
1352
1353         /* decrease freelist refcount */
1354         OSLockAcquire(psDevInfo->hLockFreeList);
1355         for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
1356         {
1357                 PVR_ASSERT(psCleanupData->apsFreeLists[ui32Loop]->ui32RefCount > 0);
1358                 psCleanupData->apsFreeLists[ui32Loop]->ui32RefCount--;
1359         }
1360         OSLockRelease(psDevInfo->hLockFreeList);
1361
1362         OSFreeMem(psCleanupData);
1363
1364         return PVRSRV_OK;
1365 }
1366
1367 IMG_EXPORT
1368 PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA      *psConnection,
1369                                PVRSRV_DEVICE_NODE       *psDeviceNode,
1370                                                            IMG_UINT32                   ui32MaxFLPages,
1371                                                            IMG_UINT32                   ui32InitFLPages,
1372                                                            IMG_UINT32                   ui32GrowFLPages,
1373                                                            RGX_FREELIST                 *psGlobalFreeList,
1374                                                            IMG_BOOL                             bCheckFreelist,
1375                                                            IMG_DEV_VIRTADDR             sFreeListDevVAddr,
1376                                                            PMR                                  *psFreeListPMR,
1377                                                            IMG_DEVMEM_OFFSET_T  uiFreeListPMROffset,
1378                                                            RGX_FREELIST                 **ppsFreeList)
1379 {
1380         PVRSRV_ERROR                            eError;
1381         RGXFWIF_FREELIST                        *psFWFreeList;
1382         DEVMEM_MEMDESC                          *psFWFreelistMemDesc;
1383         RGX_FREELIST                            *psFreeList;
1384         PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
1385
1386         /* Allocate kernel freelist struct */
1387         psFreeList = OSAllocZMem(sizeof(*psFreeList));
1388         if (psFreeList == NULL)
1389         {
1390                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateFreeList: failed to allocate host data structure"));
1391                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1392                 goto ErrorAllocHost;
1393         }
1394
1395         /* Allocate cleanup sync */
1396         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
1397                                                    &psFreeList->psCleanupSync,
1398                                                    "ta3d free list cleanup");
1399         if (eError != PVRSRV_OK)
1400         {
1401                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateFreeList: Failed to allocate cleanup sync (0x%x)",
1402                                 eError));
1403                 goto SyncAlloc;
1404         }
1405
1406         /*
1407          * This FW FreeList context is only mapped into kernel for initialisation
1408          * and reconstruction (at other times it is not mapped and only used by
1409          * the FW. Therefore the GPU cache doesn't need coherency, and write-combine
1410          * is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
1411          */
1412         eError = DevmemFwAllocate(psDevInfo,
1413                                                         sizeof(*psFWFreeList),
1414                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1415                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1416                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1417                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1418                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1419                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1420                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1421                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1422                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1423                                                         "FwFreeList",
1424                                                         &psFWFreelistMemDesc);
1425         if (eError != PVRSRV_OK)
1426         {
1427                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateFreeList: DevmemAllocate for RGXFWIF_FREELIST failed"));
1428                 goto FWFreeListAlloc;
1429         }
1430
1431         /* Initialise host data structures */
1432         psFreeList->psDevInfo = psDevInfo;
1433         psFreeList->psFreeListPMR = psFreeListPMR;
1434         psFreeList->uiFreeListPMROffset = uiFreeListPMROffset;
1435         psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
1436         RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1437         psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
1438         psFreeList->ui32FreelistGlobalID = (psGlobalFreeList ? psGlobalFreeList->ui32FreelistID : 0);
1439         psFreeList->ui32MaxFLPages = ui32MaxFLPages;
1440         psFreeList->ui32InitFLPages = ui32InitFLPages;
1441         psFreeList->ui32GrowFLPages = ui32GrowFLPages;
1442         psFreeList->ui32CurrentFLPages = 0;
1443         psFreeList->ui64FreelistChecksum = 0;
1444         psFreeList->ui32RefCount = 0;
1445         psFreeList->bCheckFreelist = bCheckFreelist;
1446         dllist_init(&psFreeList->sMemoryBlockHead);
1447         dllist_init(&psFreeList->sMemoryBlockInitHead);
1448
1449
1450         /* Add to list of freelists */
1451         OSLockAcquire(psDevInfo->hLockFreeList);
1452         dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
1453         OSLockRelease(psDevInfo->hLockFreeList);
1454
1455
1456         /* Initialise FW data structure */
1457         eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
1458         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
1459
1460         psFWFreeList->ui32MaxPages = ui32MaxFLPages;
1461         psFWFreeList->ui32CurrentPages = ui32InitFLPages;
1462         psFWFreeList->ui32GrowPages = ui32GrowFLPages;
1463         psFWFreeList->ui32CurrentStackTop = ui32InitFLPages - 1;
1464         psFWFreeList->psFreeListDevVAddr = sFreeListDevVAddr;
1465         psFWFreeList->ui64CurrentDevVAddr = sFreeListDevVAddr.uiAddr + ((ui32MaxFLPages - ui32InitFLPages) * sizeof(IMG_UINT32));
1466         psFWFreeList->ui32FreeListID = psFreeList->ui32FreelistID;
1467         psFWFreeList->bGrowPending = IMG_FALSE;
1468
1469         PVR_DPF((PVR_DBG_MESSAGE,"Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, Max FL base address 0x%016llx, Init FL base address 0x%016llx",
1470                         psFreeList,
1471                         ui32MaxFLPages,
1472                         ui32InitFLPages,
1473                         sFreeListDevVAddr.uiAddr,
1474                         psFWFreeList->psFreeListDevVAddr.uiAddr));
1475
1476         PDUMPCOMMENT("Dump FW FreeList");
1477         DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
1478
1479         /*
1480          * Separate dump of the Freelist's number of Pages and stack pointer.
1481          * This allows to easily modify the PB size in the out2.txt files.
1482          */
1483         PDUMPCOMMENT("FreeList TotalPages");
1484         DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
1485                                                         offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
1486                                                         psFWFreeList->ui32CurrentPages,
1487                                                         PDUMP_FLAGS_CONTINUOUS);
1488         PDUMPCOMMENT("FreeList StackPointer");
1489         DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
1490                                                         offsetof(RGXFWIF_FREELIST, ui32CurrentStackTop),
1491                                                         psFWFreeList->ui32CurrentStackTop,
1492                                                         PDUMP_FLAGS_CONTINUOUS);
1493
1494         DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
1495
1496
1497         /* Add initial PB block */
1498         eError = RGXGrowFreeList(psFreeList,
1499                                                                 ui32InitFLPages,
1500                                                                 &psFreeList->sMemoryBlockInitHead);
1501         if (eError != PVRSRV_OK)
1502         {
1503                 PVR_DPF((PVR_DBG_ERROR,
1504                                 "RGXCreateFreeList: failed to allocate initial memory block for free list 0x%016llx (error = %u)",
1505                                 sFreeListDevVAddr.uiAddr,
1506                                 eError));
1507                 goto FWFreeListCpuMap;
1508         }
1509 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
1510                         /* Update Stats */
1511                         PVRSRVStatsUpdateFreelistStats(1, /* Add 1 to the appropriate counter (Requests by App)*/
1512                                        0,
1513                                        psFreeList->ui32InitFLPages,
1514                                        psFreeList->ui32NumHighPages,
1515                                        psFreeList->ownerPid);
1516
1517 #endif
1518
1519         psFreeList->ownerPid = OSGetCurrentClientProcessIDKM();
1520         /* return values */
1521         *ppsFreeList = psFreeList;
1522
1523         return PVRSRV_OK;
1524
1525         /* Error handling */
1526
1527 FWFreeListCpuMap:
1528         /* Remove freelists from list  */
1529         OSLockAcquire(psDevInfo->hLockFreeList);
1530         dllist_remove_node(&psFreeList->sNode);
1531         OSLockRelease(psDevInfo->hLockFreeList);
1532
1533         RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
1534         DevmemFwFree(psDevInfo, psFWFreelistMemDesc);
1535
1536 FWFreeListAlloc:
1537         SyncPrimFree(psFreeList->psCleanupSync);
1538
1539 SyncAlloc:
1540         OSFreeMem(psFreeList);
1541
1542 ErrorAllocHost:
1543         PVR_ASSERT(eError != PVRSRV_OK);
1544         return eError;
1545 }
1546
1547
1548 /*
1549         RGXDestroyFreeList
1550 */
1551 IMG_EXPORT
1552 PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
1553 {
1554         PVRSRV_ERROR eError;
1555
1556         PVR_ASSERT(psFreeList);
1557
1558         if (psFreeList->ui32RefCount != 0)
1559         {
1560                 /* Freelist still busy */
1561                 return PVRSRV_ERROR_RETRY;
1562         }
1563
1564         /* Freelist is not in use => start firmware cleanup */
1565         eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
1566                                                                                  psFreeList->sFreeListFWDevVAddr,
1567                                                                                  psFreeList->psCleanupSync);
1568         if(eError != PVRSRV_OK)
1569         {
1570                 /* Can happen if the firmware took too long to handle the cleanup request,
1571                  * or if SLC-flushes didn't went through (due to some GPU lockup) */
1572                 return eError;
1573         }
1574
1575         /* Remove FreeList from linked list before we destroy it... */
1576         OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
1577         dllist_remove_node(&psFreeList->sNode);
1578         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
1579
1580         if (psFreeList->bCheckFreelist)
1581         {
1582                 RGXFWIF_FREELIST  *psFWFreeList;
1583                 IMG_UINT64        ui32CurrentStackTop;
1584                 IMG_UINT64        ui64CheckSum;
1585
1586                 /* Get the current stack pointer for this free list */
1587                 DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
1588                 ui32CurrentStackTop = psFWFreeList->ui32CurrentStackTop;
1589                 DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
1590
1591                 if (ui32CurrentStackTop == psFreeList->ui32CurrentFLPages-1)
1592                 {
1593                         /* Do consistency tests (as the list is fully populated) */
1594                         _CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
1595                 }
1596                 else
1597                 {
1598                         /* Check for duplicate pages, but don't check the checksum as the list is not fully populated */
1599                         _CheckFreelist(psFreeList, ui32CurrentStackTop+1, 0, &ui64CheckSum);
1600                 }
1601         }
1602
1603         /* Destroy FW structures */
1604         RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
1605         DevmemFwFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);
1606
1607         /* Remove grow shrink blocks */
1608         while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
1609         {
1610                 eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
1611                 PVR_ASSERT(eError == PVRSRV_OK);
1612         }
1613
1614         /* Remove initial PB block */
1615         eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
1616         PVR_ASSERT(eError == PVRSRV_OK);
1617
1618         /* consistency checks */
1619         PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
1620         PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
1621
1622         SyncPrimFree(psFreeList->psCleanupSync);
1623
1624         /* free Freelist */
1625         OSFreeMem(psFreeList);
1626
1627         return eError;
1628 }
1629
1630
1631
1632 /*
1633         RGXAddBlockToFreeListKM
1634 */
1635
1636 IMG_EXPORT
1637 PVRSRV_ERROR RGXAddBlockToFreeListKM(RGX_FREELIST *psFreeList,
1638                                                                                 IMG_UINT32 ui32NumPages)
1639 {
1640         PVRSRV_ERROR eError;
1641
1642         /* Check if we have reference to freelist's PMR */
1643         if (psFreeList->psFreeListPMR == NULL)
1644         {
1645                 PVR_DPF((PVR_DBG_ERROR, "Freelist is not configured for grow"));
1646                 return PVRSRV_ERROR_INVALID_PARAMS;
1647         }
1648
1649         /* grow freelist */
1650         eError = RGXGrowFreeList(psFreeList,
1651                                                         ui32NumPages,
1652                                                         &psFreeList->sMemoryBlockHead);
1653         if(eError == PVRSRV_OK)
1654         {
1655                 /* update freelist data in firmware */
1656                 _UpdateFwFreelistSize(psFreeList, IMG_TRUE, ui32NumPages);
1657
1658                 psFreeList->ui32NumGrowReqByApp++;
1659
1660 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
1661                         /* Update Stats */
1662                         PVRSRVStatsUpdateFreelistStats(1, /* Add 1 to the appropriate counter (Requests by App)*/
1663                                        0,
1664                                        psFreeList->ui32InitFLPages,
1665                                        psFreeList->ui32NumHighPages,
1666                                        psFreeList->ownerPid);
1667
1668 #endif
1669         }
1670
1671         return eError;
1672 }
1673
1674 /*
1675         RGXRemoveBlockFromFreeListKM
1676 */
1677
1678 IMG_EXPORT
1679 PVRSRV_ERROR RGXRemoveBlockFromFreeListKM(RGX_FREELIST *psFreeList)
1680 {
1681         PVRSRV_ERROR eError;
1682
1683         /*
1684          * Make sure the pages part of the memory block are not in use anymore.
1685          * Instruct the firmware to update the freelist pointers accordingly.
1686          */
1687
1688         eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead,
1689                                                                 psFreeList);
1690
1691         return eError;
1692 }
1693
1694
1695 /*
1696         RGXCreateRenderTarget
1697 */
1698 IMG_EXPORT
1699 PVRSRV_ERROR RGXCreateRenderTarget(CONNECTION_DATA      *psConnection,
1700                                    PVRSRV_DEVICE_NODE   *psDeviceNode,
1701                                                                    IMG_DEV_VIRTADDR             psVHeapTableDevVAddr,
1702                                                                    RGX_RT_CLEANUP_DATA  **ppsCleanupData,
1703                                                                    IMG_UINT32                   *sRenderTargetFWDevVAddr)
1704 {
1705         PVRSRV_ERROR                    eError = PVRSRV_OK;
1706         RGXFWIF_RENDER_TARGET   *psRenderTarget;
1707         RGXFWIF_DEV_VIRTADDR    pFirmwareAddr;
1708         PVRSRV_RGXDEV_INFO              *psDevInfo = psDeviceNode->pvDevice;
1709         RGX_RT_CLEANUP_DATA             *psCleanupData;
1710
1711         PVR_UNREFERENCED_PARAMETER(psConnection);
1712
1713         psCleanupData = OSAllocZMem(sizeof(*psCleanupData));
1714         if (psCleanupData == NULL)
1715         {
1716                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1717                 goto err_out;
1718         }
1719
1720         psCleanupData->psDeviceNode = psDeviceNode;
1721         /*
1722          * This FW render target context is only mapped into kernel for initialisation.
1723          * Otherwise this allocation is only used by the FW.
1724          * Therefore the GPU cache doesn't need coherency,
1725          * and write-combine is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
1726          */
1727         eError = DevmemFwAllocate(psDevInfo,
1728                                                         sizeof(*psRenderTarget),
1729                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1730                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1731                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1732                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1733                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1734                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1735                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1736                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1737                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1738                                                         "FwRenderTarget",
1739                                                         &psCleanupData->psRenderTargetMemDesc);
1740         if (eError != PVRSRV_OK)
1741         {
1742                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateRenderTarget: DevmemAllocate for Render Target failed"));
1743                 goto err_free;
1744         }
1745         RGXSetFirmwareAddress(&pFirmwareAddr, psCleanupData->psRenderTargetMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1746         *sRenderTargetFWDevVAddr = pFirmwareAddr.ui32Addr;
1747
1748         eError = DevmemAcquireCpuVirtAddr(psCleanupData->psRenderTargetMemDesc, (void **)&psRenderTarget);
1749         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", err_fwalloc);
1750
1751         psRenderTarget->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
1752         psRenderTarget->bTACachesNeedZeroing = IMG_FALSE;
1753         PDUMPCOMMENT("Dump RenderTarget");
1754         DevmemPDumpLoadMem(psCleanupData->psRenderTargetMemDesc, 0, sizeof(*psRenderTarget), PDUMP_FLAGS_CONTINUOUS);
1755         DevmemReleaseCpuVirtAddr(psCleanupData->psRenderTargetMemDesc);
1756
1757         *ppsCleanupData = psCleanupData;
1758
1759 err_out:
1760         return eError;
1761
1762 err_free:
1763         OSFreeMem(psCleanupData);
1764         goto err_out;
1765
1766 err_fwalloc:
1767         DevmemFwFree(psDevInfo, psCleanupData->psRenderTargetMemDesc);
1768         goto err_free;
1769
1770 }
1771
1772
1773 /*
1774         RGXDestroyRenderTarget
1775 */
1776 IMG_EXPORT
1777 PVRSRV_ERROR RGXDestroyRenderTarget(RGX_RT_CLEANUP_DATA *psCleanupData)
1778 {
1779         PVRSRV_DEVICE_NODE *psDeviceNode = psCleanupData->psDeviceNode;
1780
1781         RGXUnsetFirmwareAddress(psCleanupData->psRenderTargetMemDesc);
1782
1783         /*
1784                 Note:
1785                 When we get RT cleanup in the FW call that instead
1786         */
1787         /* Flush the the SLC before freeing */
1788         {
1789                 RGXFWIF_KCCB_CMD sFlushInvalCmd;
1790                 PVRSRV_ERROR eError;
1791
1792                 /* Schedule the SLC flush command ... */
1793 #if defined(PDUMP)
1794                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Submit SLC flush and invalidate");
1795 #endif
1796                 sFlushInvalCmd.eCmdType = RGXFWIF_KCCB_CMD_SLCFLUSHINVAL;
1797                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.bInval = IMG_TRUE;
1798                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.bDMContext = IMG_FALSE;
1799                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.eDM = 0;
1800                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.psContext.ui32Addr = 0;
1801
1802                 eError = RGXSendCommandWithPowLock(psDeviceNode->pvDevice,
1803                                                                                         RGXFWIF_DM_GP,
1804                                                                                         &sFlushInvalCmd,
1805                                                                                         sizeof(sFlushInvalCmd),
1806                                                                                         PDUMP_FLAGS_CONTINUOUS);
1807                 if (eError != PVRSRV_OK)
1808                 {
1809                         PVR_DPF((PVR_DBG_ERROR,"RGXDestroyRenderTarget: Failed to schedule SLC flush command with error (%u)", eError));
1810                 }
1811                 else
1812                 {
1813                         /* Wait for the SLC flush to complete */
1814                         eError = RGXWaitForFWOp(psDeviceNode->pvDevice, RGXFWIF_DM_GP, psDeviceNode->psSyncPrim, PDUMP_FLAGS_CONTINUOUS);
1815                         if (eError != PVRSRV_OK)
1816                         {
1817                                 PVR_DPF((PVR_DBG_ERROR,"RGXDestroyRenderTarget: SLC flush and invalidate aborted with error (%u)", eError));
1818                         }
1819                 }
1820         }
1821
1822         DevmemFwFree(psDeviceNode->pvDevice, psCleanupData->psRenderTargetMemDesc);
1823         OSFreeMem(psCleanupData);
1824         return PVRSRV_OK;
1825 }
1826
1827 /*
1828         RGXCreateZSBuffer
1829 */
1830 IMG_EXPORT
1831 PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
1832                                  PVRSRV_DEVICE_NODE     *psDeviceNode,
1833                                  DEVMEMINT_RESERVATION  *psReservation,
1834                                  PMR                                    *psPMR,
1835                                  PVRSRV_MEMALLOCFLAGS_T         uiMapFlags,
1836                                  RGX_ZSBUFFER_DATA **ppsZSBuffer,
1837                                  IMG_UINT32 *pui32ZSBufferFWDevVAddr)
1838 {
1839         PVRSRV_ERROR                            eError;
1840         PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
1841         RGXFWIF_FWZSBUFFER                      *psFWZSBuffer;
1842         RGX_ZSBUFFER_DATA                       *psZSBuffer;
1843         DEVMEM_MEMDESC                          *psFWZSBufferMemDesc;
1844         IMG_BOOL                                        bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiMapFlags) ? IMG_TRUE : IMG_FALSE;
1845
1846         /* Allocate host data structure */
1847         psZSBuffer = OSAllocZMem(sizeof(*psZSBuffer));
1848         if (psZSBuffer == NULL)
1849         {
1850                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to allocate cleanup data structure for ZS-Buffer"));
1851                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1852                 goto ErrorAllocCleanup;
1853         }
1854
1855         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
1856                                                    &psZSBuffer->psCleanupSync,
1857                                                    "ta3d zs buffer cleanup");
1858         if (eError != PVRSRV_OK)
1859         {
1860                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to allocate cleanup sync (0x%x)",
1861                                 eError));
1862                 goto ErrorSyncAlloc;
1863         }
1864
1865         /* Populate Host data */
1866         psZSBuffer->psDevInfo = psDevInfo;
1867         psZSBuffer->psReservation = psReservation;
1868         psZSBuffer->psPMR = psPMR;
1869         psZSBuffer->uiMapFlags = uiMapFlags;
1870         psZSBuffer->ui32RefCount = 0;
1871         psZSBuffer->bOnDemand = bOnDemand;
1872     if (bOnDemand)
1873     {
1874         psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
1875                 psZSBuffer->psMapping = NULL;
1876
1877                 OSLockAcquire(psDevInfo->hLockZSBuffer);
1878         dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
1879                 OSLockRelease(psDevInfo->hLockZSBuffer);
1880     }
1881
1882         /* Allocate firmware memory for ZS-Buffer. */
1883         PDUMPCOMMENT("Allocate firmware ZS-Buffer data structure");
1884         eError = DevmemFwAllocate(psDevInfo,
1885                                                         sizeof(*psFWZSBuffer),
1886                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1887                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1888                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1889                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1890                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1891                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1892                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1893                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1894                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1895                                                         "FwZSBuffer",
1896                                                         &psFWZSBufferMemDesc);
1897         if (eError != PVRSRV_OK)
1898         {
1899                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to allocate firmware ZS-Buffer (%u)", eError));
1900                 goto ErrorAllocFWZSBuffer;
1901         }
1902         psZSBuffer->psZSBufferMemDesc = psFWZSBufferMemDesc;
1903
1904         /* Temporarily map the firmware render context to the kernel. */
1905         eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
1906                                       (void **)&psFWZSBuffer);
1907         if (eError != PVRSRV_OK)
1908         {
1909                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to map firmware ZS-Buffer (%u)", eError));
1910                 goto ErrorAcquireFWZSBuffer;
1911         }
1912
1913         /* Populate FW ZS-Buffer data structure */
1914         psFWZSBuffer->bOnDemand = bOnDemand;
1915         psFWZSBuffer->eState = (bOnDemand) ? RGXFWIF_ZSBUFFER_UNBACKED : RGXFWIF_ZSBUFFER_BACKED;
1916         psFWZSBuffer->ui32ZSBufferID = psZSBuffer->ui32ZSBufferID;
1917
1918         /* Get firmware address of ZS-Buffer. */
1919         RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1920
1921         /* Dump the ZS-Buffer and the memory content */
1922         PDUMPCOMMENT("Dump firmware ZS-Buffer");
1923         DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
1924
1925         /* Release address acquired above. */
1926         DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
1927
1928
1929         /* define return value */
1930         *ppsZSBuffer = psZSBuffer;
1931         *pui32ZSBufferFWDevVAddr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
1932
1933         PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
1934                                                         psZSBuffer,
1935                                                         (bOnDemand) ? "On-Demand": "Up-front"));
1936
1937         psZSBuffer->owner=OSGetCurrentClientProcessIDKM();
1938
1939         return PVRSRV_OK;
1940
1941         /* error handling */
1942
1943 ErrorAcquireFWZSBuffer:
1944         DevmemFwFree(psDevInfo, psFWZSBufferMemDesc);
1945
1946 ErrorAllocFWZSBuffer:
1947         SyncPrimFree(psZSBuffer->psCleanupSync);
1948
1949 ErrorSyncAlloc:
1950         OSFreeMem(psZSBuffer);
1951
1952 ErrorAllocCleanup:
1953         PVR_ASSERT(eError != PVRSRV_OK);
1954         return eError;
1955 }
1956
1957
1958 /*
1959         RGXDestroyZSBuffer
1960 */
1961 IMG_EXPORT
1962 PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
1963 {
1964         POS_LOCK hLockZSBuffer;
1965         PVRSRV_ERROR eError;
1966
1967         PVR_ASSERT(psZSBuffer);
1968         hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
1969
1970         /* Request ZS Buffer cleanup */
1971         eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
1972                                                                                 psZSBuffer->sZSBufferFWDevVAddr,
1973                                                                                 psZSBuffer->psCleanupSync);
1974         if (eError != PVRSRV_ERROR_RETRY)
1975         {
1976                 /* Free the firmware render context. */
1977         RGXUnsetFirmwareAddress(psZSBuffer->psZSBufferMemDesc);
1978                 DevmemFwFree(psZSBuffer->psDevInfo, psZSBuffer->psZSBufferMemDesc);
1979
1980             /* Remove Deferred Allocation from list */
1981                 if (psZSBuffer->bOnDemand)
1982                 {
1983                         OSLockAcquire(hLockZSBuffer);
1984                         PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
1985                         dllist_remove_node(&psZSBuffer->sNode);
1986                         OSLockRelease(hLockZSBuffer);
1987                 }
1988
1989                 SyncPrimFree(psZSBuffer->psCleanupSync);
1990
1991                 PVR_ASSERT(psZSBuffer->ui32RefCount == 0);
1992
1993                 PVR_DPF((PVR_DBG_MESSAGE,"ZS-Buffer [%p] destroyed",psZSBuffer));
1994
1995                 /* Free ZS-Buffer host data structure */
1996                 OSFreeMem(psZSBuffer);
1997
1998         }
1999
2000         return eError;
2001 }
2002
2003 PVRSRV_ERROR
2004 RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
2005 {
2006         POS_LOCK hLockZSBuffer;
2007         PVRSRV_ERROR eError;
2008
2009         if (!psZSBuffer)
2010         {
2011                 return PVRSRV_ERROR_INVALID_PARAMS;
2012         }
2013
2014         if (!psZSBuffer->bOnDemand)
2015         {
2016                 /* Only deferred allocations can be populated */
2017                 return PVRSRV_ERROR_INVALID_PARAMS;
2018         }
2019
2020         PVR_DPF((PVR_DBG_MESSAGE,"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
2021                                                                 psZSBuffer,
2022                                                                 psZSBuffer->ui32ZSBufferID));
2023         hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
2024
2025         OSLockAcquire(hLockZSBuffer);
2026
2027         if (psZSBuffer->ui32RefCount == 0)
2028         {
2029                 if (psZSBuffer->bOnDemand)
2030                 {
2031                         IMG_HANDLE hDevmemHeap;
2032
2033                         PVR_ASSERT(psZSBuffer->psMapping == NULL);
2034
2035                         /* Get Heap */
2036                         eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
2037                         PVR_ASSERT(psZSBuffer->psMapping == NULL);
2038
2039                         eError = DevmemIntMapPMR(hDevmemHeap,
2040                                                                         psZSBuffer->psReservation,
2041                                                                         psZSBuffer->psPMR,
2042                                                                         psZSBuffer->uiMapFlags,
2043                                                                         &psZSBuffer->psMapping);
2044                         if (eError != PVRSRV_OK)
2045                         {
2046                                 PVR_DPF((PVR_DBG_ERROR,"Unable populate ZS Buffer [%p, ID=0x%08x] with error %u",
2047                                                                                 psZSBuffer,
2048                                                                                 psZSBuffer->ui32ZSBufferID,
2049                                                                                 eError));
2050                                 OSLockRelease(hLockZSBuffer);
2051                                 return eError;
2052
2053                         }
2054                         PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
2055                                                                                 psZSBuffer,
2056                                                                                 psZSBuffer->ui32ZSBufferID));
2057                 }
2058         }
2059
2060         /* Increase refcount*/
2061         psZSBuffer->ui32RefCount++;
2062
2063         OSLockRelease(hLockZSBuffer);
2064
2065         return PVRSRV_OK;
2066 }
2067
2068
2069 PVRSRV_ERROR
2070 RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
2071                                         RGX_POPULATION **ppsPopulation)
2072 {
2073         RGX_POPULATION *psPopulation;
2074         PVRSRV_ERROR eError;
2075
2076         psZSBuffer->ui32NumReqByApp++;
2077
2078 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
2079         PVRSRVStatsUpdateZSBufferStats(1,0,psZSBuffer->owner);
2080 #endif
2081
2082         /* Do the backing */
2083         eError = RGXBackingZSBuffer(psZSBuffer);
2084         if (eError != PVRSRV_OK)
2085         {
2086                 goto OnErrorBacking;
2087         }
2088
2089         /* Create the handle to the backing */
2090         psPopulation = OSAllocMem(sizeof(*psPopulation));
2091         if (psPopulation == NULL)
2092         {
2093                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
2094                 goto OnErrorAlloc;
2095         }
2096
2097         psPopulation->psZSBuffer = psZSBuffer;
2098
2099         /* return value */
2100         *ppsPopulation = psPopulation;
2101
2102         return PVRSRV_OK;
2103
2104 OnErrorAlloc:
2105         RGXUnbackingZSBuffer(psZSBuffer);
2106
2107 OnErrorBacking:
2108         PVR_ASSERT(eError != PVRSRV_OK);
2109         return eError;
2110 }
2111
2112 PVRSRV_ERROR
2113 RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
2114 {
2115         POS_LOCK hLockZSBuffer;
2116         PVRSRV_ERROR eError;
2117
2118         if (!psZSBuffer)
2119         {
2120                 return PVRSRV_ERROR_INVALID_PARAMS;
2121         }
2122
2123         PVR_ASSERT(psZSBuffer->ui32RefCount);
2124
2125         PVR_DPF((PVR_DBG_MESSAGE,"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
2126                                                                 psZSBuffer,
2127                                                                 psZSBuffer->ui32ZSBufferID));
2128
2129         hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
2130
2131         OSLockAcquire(hLockZSBuffer);
2132
2133         if (psZSBuffer->bOnDemand)
2134         {
2135                 if (psZSBuffer->ui32RefCount == 1)
2136                 {
2137                         PVR_ASSERT(psZSBuffer->psMapping);
2138
2139                         eError = DevmemIntUnmapPMR(psZSBuffer->psMapping);
2140                         if (eError != PVRSRV_OK)
2141                         {
2142                                 PVR_DPF((PVR_DBG_ERROR,"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] with error %u",
2143                                                                                 psZSBuffer,
2144                                                                                 psZSBuffer->ui32ZSBufferID,
2145                                                                                 eError));
2146                                 OSLockRelease(hLockZSBuffer);
2147                                 return eError;
2148                         }
2149
2150                         PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
2151                                                                                 psZSBuffer,
2152                                                                                 psZSBuffer->ui32ZSBufferID));
2153                 }
2154         }
2155
2156         /* Decrease refcount*/
2157         psZSBuffer->ui32RefCount--;
2158
2159         OSLockRelease(hLockZSBuffer);
2160
2161         return PVRSRV_OK;
2162 }
2163
2164 PVRSRV_ERROR
2165 RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
2166 {
2167         PVRSRV_ERROR eError;
2168
2169         if (!psPopulation)
2170         {
2171                 return PVRSRV_ERROR_INVALID_PARAMS;
2172         }
2173
2174         eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
2175         if (eError != PVRSRV_OK)
2176         {
2177                 return eError;
2178         }
2179
2180         OSFreeMem(psPopulation);
2181
2182         return PVRSRV_OK;
2183 }
2184
2185 static RGX_ZSBUFFER_DATA *FindZSBuffer(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32ZSBufferID)
2186 {
2187         DLLIST_NODE *psNode, *psNext;
2188         RGX_ZSBUFFER_DATA *psZSBuffer = NULL;
2189
2190         OSLockAcquire(psDevInfo->hLockZSBuffer);
2191
2192         dllist_foreach_node(&psDevInfo->sZSBufferHead, psNode, psNext)
2193         {
2194                 RGX_ZSBUFFER_DATA *psThisZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
2195
2196                 if (psThisZSBuffer->ui32ZSBufferID == ui32ZSBufferID)
2197                 {
2198                         psZSBuffer = psThisZSBuffer;
2199                         break;
2200                 }
2201         }
2202
2203         OSLockRelease(psDevInfo->hLockZSBuffer);
2204         return psZSBuffer;
2205 }
2206
2207 void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
2208                                                                           IMG_UINT32 ui32ZSBufferID)
2209 {
2210         RGX_ZSBUFFER_DATA *psZSBuffer;
2211         RGXFWIF_KCCB_CMD sTACCBCmd;
2212         PVRSRV_ERROR eError;
2213
2214         PVR_ASSERT(psDevInfo);
2215
2216         /* scan all deferred allocations */
2217         psZSBuffer = FindZSBuffer(psDevInfo, ui32ZSBufferID);
2218
2219         if (psZSBuffer)
2220         {
2221                 IMG_BOOL bBackingDone = IMG_TRUE;
2222
2223                 /* Populate ZLS */
2224                 eError = RGXBackingZSBuffer(psZSBuffer);
2225                 if (eError != PVRSRV_OK)
2226                 {
2227                         PVR_DPF((PVR_DBG_ERROR,"Populating ZS-Buffer failed with error %u (ID = 0x%08x)", eError, ui32ZSBufferID));
2228                         bBackingDone = IMG_FALSE;
2229                 }
2230
2231                 /* send confirmation */
2232                 sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
2233                 sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
2234                 sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
2235
2236                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
2237                 {
2238                         eError = RGXScheduleCommand(psDevInfo,
2239                                                                                 RGXFWIF_DM_TA,
2240                                                                                 &sTACCBCmd,
2241                                                                                 sizeof(sTACCBCmd),
2242                                                                                 0,
2243                                                                                 PDUMP_FLAGS_NONE);
2244                         if (eError != PVRSRV_ERROR_RETRY)
2245                         {
2246                                 break;
2247                         }
2248                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
2249                 } END_LOOP_UNTIL_TIMEOUT();
2250
2251                 /* Kernel CCB should never fill up, as the FW is processing them right away  */
2252                 PVR_ASSERT(eError == PVRSRV_OK);
2253
2254                 psZSBuffer->ui32NumReqByFW++;
2255
2256 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
2257                 PVRSRVStatsUpdateZSBufferStats(0,1,psZSBuffer->owner);
2258 #endif
2259
2260         }
2261         else
2262         {
2263                 PVR_DPF((PVR_DBG_ERROR,"ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)", ui32ZSBufferID));
2264         }
2265 }
2266
2267 void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
2268                                                                                         IMG_UINT32 ui32ZSBufferID)
2269 {
2270         RGX_ZSBUFFER_DATA *psZSBuffer;
2271         RGXFWIF_KCCB_CMD sTACCBCmd;
2272         PVRSRV_ERROR eError;
2273
2274         PVR_ASSERT(psDevInfo);
2275
2276         /* scan all deferred allocations */
2277         psZSBuffer = FindZSBuffer(psDevInfo, ui32ZSBufferID);
2278
2279         if (psZSBuffer)
2280         {
2281                 /* Unpopulate ZLS */
2282                 eError = RGXUnbackingZSBuffer(psZSBuffer);
2283                 if (eError != PVRSRV_OK)
2284                 {
2285                         PVR_DPF((PVR_DBG_ERROR,"UnPopulating ZS-Buffer failed failed with error %u (ID = 0x%08x)", eError, ui32ZSBufferID));
2286                         PVR_ASSERT(IMG_FALSE);
2287                 }
2288
2289                 /* send confirmation */
2290                 sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
2291                 sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
2292                 sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
2293
2294                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
2295                 {
2296                         eError = RGXScheduleCommand(psDevInfo,
2297                                                                                 RGXFWIF_DM_TA,
2298                                                                                 &sTACCBCmd,
2299                                                                                 sizeof(sTACCBCmd),
2300                                                                                 0,
2301                                                                                 PDUMP_FLAGS_NONE);
2302                         if (eError != PVRSRV_ERROR_RETRY)
2303                         {
2304                                 break;
2305                         }
2306                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
2307                 } END_LOOP_UNTIL_TIMEOUT();
2308
2309                 /* Kernel CCB should never fill up, as the FW is processing them right away  */
2310                 PVR_ASSERT(eError == PVRSRV_OK);
2311
2312         }
2313         else
2314         {
2315                 PVR_DPF((PVR_DBG_ERROR,"ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)", ui32ZSBufferID));
2316         }
2317 }
2318
2319 static
2320 PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
2321                                                           PVRSRV_DEVICE_NODE *psDeviceNode,
2322                                                           DEVMEM_MEMDESC *psAllocatedMemDesc,
2323                                                           IMG_UINT32 ui32AllocatedOffset,
2324                                                           DEVMEM_MEMDESC *psFWMemContextMemDesc,
2325                                                           IMG_DEV_VIRTADDR sVDMCallStackAddr,
2326                                                           IMG_UINT32 ui32Priority,
2327                                                           RGX_COMMON_CONTEXT_INFO *psInfo,
2328                                                           RGX_SERVER_RC_TA_DATA *psTAData)
2329 {
2330         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
2331         RGXFWIF_TACTX_STATE *psContextState;
2332         PVRSRV_ERROR eError;
2333         /*
2334                 Allocate device memory for the firmware GPU context suspend state.
2335                 Note: the FW reads/writes the state to memory by accessing the GPU register interface.
2336         */
2337         PDUMPCOMMENT("Allocate RGX firmware TA context suspend state");
2338
2339         eError = DevmemFwAllocate(psDevInfo,
2340                                                           sizeof(RGXFWIF_TACTX_STATE),
2341                                                           RGX_FWCOMCTX_ALLOCFLAGS,
2342                                                           "FwTAContextState",
2343                                                           &psTAData->psContextStateMemDesc);
2344         if (eError != PVRSRV_OK)
2345         {
2346                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate firmware GPU context suspend state (%u)",
2347                                 eError));
2348                 goto fail_tacontextsuspendalloc;
2349         }
2350
2351         eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
2352                                       (void **)&psContextState);
2353         if (eError != PVRSRV_OK)
2354         {
2355                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to map firmware render context state (%u)",
2356                                 eError));
2357                 goto fail_suspendcpuvirtacquire;
2358         }
2359         psContextState->uTAReg_VDM_CALL_STACK_POINTER_Init = sVDMCallStackAddr.uiAddr;
2360         DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
2361
2362         eError = FWCommonContextAllocate(psConnection,
2363                                                                          psDeviceNode,
2364                                                                          REQ_TYPE_TA,
2365                                                                          RGXFWIF_DM_TA,
2366                                                                          psAllocatedMemDesc,
2367                                                                          ui32AllocatedOffset,
2368                                                                          psFWMemContextMemDesc,
2369                                                                          psTAData->psContextStateMemDesc,
2370                                                                          RGX_TA_CCB_SIZE_LOG2,
2371                                                                          ui32Priority,
2372                                                                          psInfo,
2373                                                                          &psTAData->psServerCommonContext);
2374         if (eError != PVRSRV_OK)
2375         {
2376                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to init TA fw common context (%u)",
2377                                 eError));
2378                 goto fail_tacommoncontext;
2379         }
2380
2381         /*
2382          * Dump the FW 3D context suspend state buffer
2383          */
2384         PDUMPCOMMENT("Dump the TA context suspend state buffer");
2385         DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
2386                                            0,
2387                                            sizeof(RGXFWIF_TACTX_STATE),
2388                                            PDUMP_FLAGS_CONTINUOUS);
2389
2390         psTAData->ui32Priority = ui32Priority;
2391         return PVRSRV_OK;
2392
2393 fail_tacommoncontext:
2394 fail_suspendcpuvirtacquire:
2395         DevmemFwFree(psDevInfo, psTAData->psContextStateMemDesc);
2396 fail_tacontextsuspendalloc:
2397         PVR_ASSERT(eError != PVRSRV_OK);
2398
2399         return eError;
2400 }
2401
2402 static
2403 PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
2404                                                           PVRSRV_DEVICE_NODE *psDeviceNode,
2405                                                           DEVMEM_MEMDESC *psAllocatedMemDesc,
2406                                                           IMG_UINT32 ui32AllocatedOffset,
2407                                                           DEVMEM_MEMDESC *psFWMemContextMemDesc,
2408                                                           IMG_UINT32 ui32Priority,
2409                                                           RGX_COMMON_CONTEXT_INFO *psInfo,
2410                                                           RGX_SERVER_RC_3D_DATA *ps3DData)
2411 {
2412         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
2413         PVRSRV_ERROR eError;
2414
2415         /*
2416                 Allocate device memory for the firmware GPU context suspend state.
2417                 Note: the FW reads/writes the state to memory by accessing the GPU register interface.
2418         */
2419         PDUMPCOMMENT("Allocate RGX firmware 3D context suspend state");
2420
2421         eError = DevmemFwAllocate(psDevInfo,
2422                                                           sizeof(RGXFWIF_3DCTX_STATE),
2423                                                           RGX_FWCOMCTX_ALLOCFLAGS,
2424                                                           "Fw3DContextState",
2425                                                           &ps3DData->psContextStateMemDesc);
2426         if (eError != PVRSRV_OK)
2427         {
2428                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate firmware GPU context suspend state (%u)",
2429                                 eError));
2430                 goto fail_3dcontextsuspendalloc;
2431         }
2432
2433         eError = FWCommonContextAllocate(psConnection,
2434                                                                          psDeviceNode,
2435                                                                          REQ_TYPE_3D,
2436                                                                          RGXFWIF_DM_3D,
2437                                                                          psAllocatedMemDesc,
2438                                                                          ui32AllocatedOffset,
2439                                                                          psFWMemContextMemDesc,
2440                                                                          ps3DData->psContextStateMemDesc,
2441                                                                          RGX_3D_CCB_SIZE_LOG2,
2442                                                                          ui32Priority,
2443                                                                          psInfo,
2444                                                                          &ps3DData->psServerCommonContext);
2445         if (eError != PVRSRV_OK)
2446         {
2447                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to init 3D fw common context (%u)",
2448                                 eError));
2449                 goto fail_3dcommoncontext;
2450         }
2451
2452         /*
2453          * Dump the FW 3D context suspend state buffer
2454          */
2455         PDUMPCOMMENT("Dump the 3D context suspend state buffer");
2456         DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
2457                                            0,
2458                                            sizeof(RGXFWIF_3DCTX_STATE),
2459                                            PDUMP_FLAGS_CONTINUOUS);
2460
2461         ps3DData->ui32Priority = ui32Priority;
2462         return PVRSRV_OK;
2463
2464 fail_3dcommoncontext:
2465         DevmemFwFree(psDevInfo, ps3DData->psContextStateMemDesc);
2466 fail_3dcontextsuspendalloc:
2467         PVR_ASSERT(eError != PVRSRV_OK);
2468
2469         return eError;
2470 }
2471
2472
2473 /*
2474  * PVRSRVRGXCreateRenderContextKM
2475  */
2476 IMG_EXPORT
2477 PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA                             *psConnection,
2478                                                                                         PVRSRV_DEVICE_NODE                      *psDeviceNode,
2479                                                                                         IMG_UINT32                                      ui32Priority,
2480                                                                                         IMG_DEV_VIRTADDR                        sMCUFenceAddr,
2481                                                                                         IMG_DEV_VIRTADDR                        sVDMCallStackAddr,
2482                                                                                         IMG_UINT32                                      ui32FrameworkRegisterSize,
2483                                                                                         IMG_PBYTE                                       pabyFrameworkRegisters,
2484                                                                                         IMG_HANDLE                                      hMemCtxPrivData,
2485                                                                                         RGX_SERVER_RENDER_CONTEXT       **ppsRenderContext)
2486 {
2487         PVRSRV_ERROR                            eError;
2488         PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
2489         RGX_SERVER_RENDER_CONTEXT       *psRenderContext;
2490         DEVMEM_MEMDESC                          *psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
2491         RGX_COMMON_CONTEXT_INFO         sInfo;
2492
2493         /* Prepare cleanup structure */
2494         *ppsRenderContext = NULL;
2495         psRenderContext = OSAllocZMem(sizeof(*psRenderContext));
2496         if (psRenderContext == NULL)
2497         {
2498                 return PVRSRV_ERROR_OUT_OF_MEMORY;
2499         }
2500
2501         psRenderContext->psDeviceNode = psDeviceNode;
2502
2503         /*
2504                 Create the FW render context, this has the TA and 3D FW common
2505                 contexts embedded within it
2506         */
2507         eError = DevmemFwAllocate(psDevInfo,
2508                                                           sizeof(RGXFWIF_FWRENDERCONTEXT),
2509                                                           RGX_FWCOMCTX_ALLOCFLAGS,
2510                                                           "FwRenderContext",
2511                                                           &psRenderContext->psFWRenderContextMemDesc);
2512         if (eError != PVRSRV_OK)
2513         {
2514                 goto fail_fwrendercontext;
2515         }
2516
2517 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2518         WorkEstRCInit(&(psRenderContext->sWorkEstData));
2519 #endif
2520
2521         /* Allocate cleanup sync */
2522         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
2523                                                    &psRenderContext->psCleanupSync,
2524                                                    "ta3d render context cleanup");
2525         if (eError != PVRSRV_OK)
2526         {
2527                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate cleanup sync (0x%x)",
2528                                 eError));
2529                 goto fail_syncalloc;
2530         }
2531
2532         /*
2533          * Create the FW framework buffer
2534          */
2535         eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
2536                                                                                 &psRenderContext->psFWFrameworkMemDesc,
2537                                                                                 ui32FrameworkRegisterSize);
2538         if (eError != PVRSRV_OK)
2539         {
2540                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate firmware GPU framework state (%u)",
2541                                 eError));
2542                 goto fail_frameworkcreate;
2543         }
2544
2545         /* Copy the Framework client data into the framework buffer */
2546         eError = PVRSRVRGXFrameworkCopyCommand(psRenderContext->psFWFrameworkMemDesc,
2547                                                                                    pabyFrameworkRegisters,
2548                                                                                    ui32FrameworkRegisterSize);
2549         if (eError != PVRSRV_OK)
2550         {
2551                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to populate the framework buffer (%u)",
2552                                 eError));
2553                 goto fail_frameworkcopy;
2554         }
2555
2556         sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
2557         sInfo.psMCUFenceAddr = &sMCUFenceAddr;
2558
2559         eError = _CreateTAContext(psConnection,
2560                                                           psDeviceNode,
2561                                                           psRenderContext->psFWRenderContextMemDesc,
2562                                                           offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
2563                                                           psFWMemContextMemDesc,
2564                                                           sVDMCallStackAddr,
2565                                                           ui32Priority,
2566                                                           &sInfo,
2567                                                           &psRenderContext->sTAData);
2568         if (eError != PVRSRV_OK)
2569         {
2570                 goto fail_tacontext;
2571         }
2572
2573         eError = _Create3DContext(psConnection,
2574                                                           psDeviceNode,
2575                                                           psRenderContext->psFWRenderContextMemDesc,
2576                                                           offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
2577                                                           psFWMemContextMemDesc,
2578                                                           ui32Priority,
2579                                                           &sInfo,
2580                                                           &psRenderContext->s3DData);
2581         if (eError != PVRSRV_OK)
2582         {
2583                 goto fail_3dcontext;
2584         }
2585
2586         SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
2587         SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
2588         SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
2589         SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
2590
2591         {
2592                 PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
2593
2594                 OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
2595                 dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
2596                 OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
2597         }
2598
2599         *ppsRenderContext= psRenderContext;
2600         return PVRSRV_OK;
2601
2602 fail_3dcontext:
2603         _DestroyTAContext(&psRenderContext->sTAData,
2604                                           psDeviceNode,
2605                                           psRenderContext->psCleanupSync);
2606 fail_tacontext:
2607 fail_frameworkcopy:
2608         DevmemFwFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
2609 fail_frameworkcreate:
2610         SyncPrimFree(psRenderContext->psCleanupSync);
2611 fail_syncalloc:
2612         DevmemFwFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
2613 fail_fwrendercontext:
2614         OSFreeMem(psRenderContext);
2615         PVR_ASSERT(eError != PVRSRV_OK);
2616
2617         return eError;
2618 }
2619
2620 /*
2621  * PVRSRVRGXDestroyRenderContextKM
2622  */
2623 IMG_EXPORT
2624 PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
2625 {
2626         PVRSRV_ERROR                            eError;
2627         PVRSRV_RGXDEV_INFO      *psDevInfo = psRenderContext->psDeviceNode->pvDevice;
2628         RGXFWIF_FWRENDERCONTEXT *psFWRenderContext;
2629 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2630         IMG_UINT32 ui32WorkEstCCBSubmitted;
2631 #endif
2632
2633         /* remove node from list before calling destroy - as destroy, if successful
2634          * will invalidate the node
2635          * must be re-added if destroy fails
2636          */
2637         OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
2638         dllist_remove_node(&(psRenderContext->sListNode));
2639         OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
2640
2641         /* Cleanup the TA if we haven't already */
2642         if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
2643         {
2644                 eError = _DestroyTAContext(&psRenderContext->sTAData,
2645                                                                    psRenderContext->psDeviceNode,
2646                                                                    psRenderContext->psCleanupSync);
2647                 if (eError == PVRSRV_OK)
2648                 {
2649                         psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
2650                 }
2651                 else
2652                 {
2653                         goto e0;
2654                 }
2655         }
2656
2657         /* Cleanup the 3D if we haven't already */
2658         if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
2659         {
2660                 eError = _Destroy3DContext(&psRenderContext->s3DData,
2661                                                                    psRenderContext->psDeviceNode,
2662                                                                    psRenderContext->psCleanupSync);
2663                 if (eError == PVRSRV_OK)
2664                 {
2665                         psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
2666                 }
2667                 else
2668                 {
2669                         goto e0;
2670                 }
2671         }
2672
2673 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2674         eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
2675                                           (void **)&psFWRenderContext);
2676         if(eError != PVRSRV_OK)
2677         {
2678                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXDestroyRenderContextKM: Failed to map firmware render context (%u)",
2679                          eError));
2680                 goto e0;
2681         }
2682
2683         ui32WorkEstCCBSubmitted = psFWRenderContext->ui32WorkEstCCBSubmitted;
2684
2685         DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
2686
2687         /* Check if all of the workload estimation CCB commands for this workload
2688          * are read
2689          */
2690         if(ui32WorkEstCCBSubmitted != psRenderContext->sWorkEstData.ui32WorkEstCCBReceived)
2691         {
2692                 eError = PVRSRV_ERROR_RETRY;
2693                 goto e0;
2694         }
2695 #endif
2696
2697         /*
2698                 Only if both TA and 3D contexts have been cleaned up can we
2699                 free the shared resources
2700         */
2701         if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
2702         {
2703
2704                 /* Update SPM statistics */
2705                 eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
2706                                               (void **)&psFWRenderContext);
2707                 if (eError == PVRSRV_OK)
2708                 {
2709                         DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
2710                 }
2711                 else
2712                 {
2713                         PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXDestroyRenderContextKM: Failed to map firmware render context (%u)",
2714                                         eError));
2715                 }
2716
2717                 /* Free the framework buffer */
2718                 DevmemFwFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
2719
2720                 /* Free the firmware render context */
2721                 DevmemFwFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
2722
2723                 /* Free the cleanup sync */
2724                 SyncPrimFree(psRenderContext->psCleanupSync);
2725
2726                 SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
2727                 SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
2728                 SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
2729                 SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
2730
2731 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2732                 WorkEstRCDeInit(&(psRenderContext->sWorkEstData),
2733                         psDevInfo);
2734 #endif
2735
2736                 OSFreeMem(psRenderContext);
2737         }
2738
2739         return PVRSRV_OK;
2740
2741 e0:
2742         OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
2743         dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
2744         OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
2745         return eError;
2746 }
2747
2748
2749 /* TODO !!! this was local on the stack, and we managed to blow the stack for the kernel.
2750  * THIS - 46 argument function needs to be sorted out.
2751  */
2752 /* 1 command for the TA */
2753 static RGX_CCB_CMD_HELPER_DATA asTACmdHelperData[1];
2754 /* Up to 3 commands for the 3D (partial render fence, partial reader, and render) */
2755 static RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[3];
2756
2757 /*
2758  * PVRSRVRGXKickTA3DKM
2759  */
2760 IMG_EXPORT
2761 PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT      *psRenderContext,
2762                                                                  IMG_UINT32                                     ui32ClientCacheOpSeqNum,
2763                                                                  IMG_UINT32                                     ui32ClientTAFenceCount,
2764                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClientTAFenceSyncPrimBlock,
2765                                                                  IMG_UINT32                                     *paui32ClientTAFenceSyncOffset,
2766                                                                  IMG_UINT32                                     *paui32ClientTAFenceValue,
2767                                                                  IMG_UINT32                                     ui32ClientTAUpdateCount,
2768                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClientTAUpdateSyncPrimBlock,
2769                                                                  IMG_UINT32                                     *paui32ClientTAUpdateSyncOffset,
2770                                                                  IMG_UINT32                                     *paui32ClientTAUpdateValue,
2771                                                                  IMG_UINT32                                     ui32ServerTASyncPrims,
2772                                                                  IMG_UINT32                                     *paui32ServerTASyncFlags,
2773                                                                  SERVER_SYNC_PRIMITIVE          **pasServerTASyncs,
2774                                                                  IMG_UINT32                                     ui32Client3DFenceCount,
2775                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClient3DFenceSyncPrimBlock,
2776                                                                  IMG_UINT32                                     *paui32Client3DFenceSyncOffset,
2777                                                                  IMG_UINT32                                     *paui32Client3DFenceValue,
2778                                                                  IMG_UINT32                                     ui32Client3DUpdateCount,
2779                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClient3DUpdateSyncPrimBlock,
2780                                                                  IMG_UINT32                                     *paui32Client3DUpdateSyncOffset,
2781                                                                  IMG_UINT32                                     *paui32Client3DUpdateValue,
2782                                                                  IMG_UINT32                                     ui32Server3DSyncPrims,
2783                                                                  IMG_UINT32                                     *paui32Server3DSyncFlags,
2784                                                                  SERVER_SYNC_PRIMITIVE          **pasServer3DSyncs,
2785                                                                  SYNC_PRIMITIVE_BLOCK                           *psPRFenceSyncPrimBlock,
2786                                                                  IMG_UINT32                                     ui32PRFenceSyncOffset,
2787                                                                  IMG_UINT32                                     ui32PRFenceValue,
2788                                                                  IMG_INT32                                      i32CheckFenceFD,
2789                                                                  IMG_INT32                                      i32UpdateTimelineFD,
2790                                                                  IMG_INT32                                      *pi32UpdateFenceFD,
2791                                                                  IMG_CHAR                                       szFenceName[32],
2792                                                                  IMG_UINT32                                     ui32TACmdSize,
2793                                                                  IMG_PBYTE                                      pui8TADMCmd,
2794                                                                  IMG_UINT32                                     ui323DPRCmdSize,
2795                                                                  IMG_PBYTE                                      pui83DPRDMCmd,
2796                                                                  IMG_UINT32                                     ui323DCmdSize,
2797                                                                  IMG_PBYTE                                      pui83DDMCmd,
2798                                                                  IMG_UINT32                                     ui32ExtJobRef,
2799                                                                  IMG_BOOL                                       bLastTAInScene,
2800                                                                  IMG_BOOL                                       bKickTA,
2801                                                                  IMG_BOOL                                       bKickPR,
2802                                                                  IMG_BOOL                                       bKick3D,
2803                                                                  IMG_BOOL                                       bAbort,
2804                                                                  IMG_UINT32                                     ui32PDumpFlags,
2805                                                                  RGX_RTDATA_CLEANUP_DATA        *psRTDataCleanup,
2806                                                                  RGX_ZSBUFFER_DATA              *psZBuffer,
2807                                                                  RGX_ZSBUFFER_DATA              *psSBuffer,
2808                                                                  IMG_BOOL                       bCommitRefCountsTA,
2809                                                                  IMG_BOOL                       bCommitRefCounts3D,
2810                                                                  IMG_BOOL                       *pbCommittedRefCountsTA,
2811                                                                  IMG_BOOL                       *pbCommittedRefCounts3D,
2812                                                                  IMG_UINT32                     ui32SyncPMRCount,
2813                                                                  IMG_UINT32                     *paui32SyncPMRFlags,
2814                                                                  PMR                            **ppsSyncPMRs,
2815                                                                  IMG_UINT32                     ui32RenderTargetSize,
2816                                                                  IMG_UINT32                     ui32NumberOfDrawCalls,
2817                                                                  IMG_UINT32                     ui32NumberOfIndices,
2818                                                                  IMG_UINT32                     ui32NumberOfMRTs,
2819                                                                  IMG_UINT64                     ui64DeadlineInus)
2820 {
2821
2822         IMG_UINT32                              ui32TACmdCount=0;
2823         IMG_UINT32                              ui323DCmdCount=0;
2824         IMG_UINT32                              ui32TACmdOffset=0;
2825         IMG_UINT32                              ui323DCmdOffset=0;
2826         RGXFWIF_UFO                             sPRUFO;
2827         IMG_UINT32                              i;
2828         PVRSRV_ERROR                    eError = PVRSRV_OK;
2829         PVRSRV_ERROR                    eError2;
2830         IMG_INT32                               i32UpdateFenceFD = -1;
2831         IMG_UINT32                              ui32JobId;
2832
2833         IMG_UINT32                              ui32ClientPRUpdateCount = 0;
2834         PRGXFWIF_UFO_ADDR               *pauiClientPRUpdateUFOAddress = NULL;
2835         IMG_UINT32                              *paui32ClientPRUpdateValue = NULL;
2836
2837         PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
2838         PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
2839         PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
2840
2841         PRGXFWIF_UFO_ADDR                       *pauiClientTAFenceUFOAddress;
2842         PRGXFWIF_UFO_ADDR                       *pauiClientTAUpdateUFOAddress;
2843         PRGXFWIF_UFO_ADDR                       *pauiClient3DFenceUFOAddress;
2844         PRGXFWIF_UFO_ADDR                       *pauiClient3DUpdateUFOAddress;
2845         PRGXFWIF_UFO_ADDR                       uiPRFenceUFOAddress;
2846
2847 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2848         RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTA;
2849         RGXFWIF_WORKEST_KICK_DATA sWorkloadKickData3D;
2850         IMG_UINT32 ui32TACommandOffset = 0;
2851         IMG_UINT32 ui323DCommandOffset = 0;
2852         IMG_UINT32 ui32TACmdHeaderOffset = 0;
2853         IMG_UINT32 ui323DCmdHeaderOffset = 0;
2854         IMG_UINT32 ui323DFullRenderCommandOffset = 0;
2855         IMG_UINT32 ui32TACmdOffsetWrapCheck = 0;
2856         IMG_UINT32 ui323DCmdOffsetWrapCheck = 0;
2857 #endif
2858
2859 #if defined(SUPPORT_BUFFER_SYNC)
2860         struct pvr_buffer_sync_append_data *psAppendData = NULL;
2861 #endif
2862
2863 #if defined(SUPPORT_NATIVE_FENCE_SYNC)
2864         /* Android fd sync update info */
2865         struct pvr_sync_append_data *psFDData = NULL;
2866         if (i32UpdateTimelineFD >= 0 && !pi32UpdateFenceFD)
2867         {
2868                 return PVRSRV_ERROR_INVALID_PARAMS;
2869         }
2870 #else
2871         if (i32UpdateTimelineFD >= 0)
2872         {
2873                 PVR_DPF((PVR_DBG_WARNING, "%s: Providing native sync timeline (%d) in non native sync enabled driver",
2874                         __func__, i32UpdateTimelineFD));
2875         }
2876         if (i32CheckFenceFD >= 0)
2877         {
2878                 PVR_DPF((PVR_DBG_WARNING, "%s: Providing native check sync (%d) in non native sync enabled driver",
2879                         __func__, i32CheckFenceFD));
2880         }
2881 #endif
2882
2883 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2884         sWorkloadKickDataTA.ui64ReturnDataIndex = 0;
2885         sWorkloadKickData3D.ui64ReturnDataIndex = 0;
2886 #endif
2887
2888         ui32JobId = OSAtomicIncrement(&psRenderContext->hJobId);
2889
2890         /* Ensure the string is null-terminated (Required for safety) */
2891         szFenceName[31] = '\0';
2892         *pbCommittedRefCountsTA = IMG_FALSE;
2893         *pbCommittedRefCounts3D = IMG_FALSE;
2894
2895         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
2896                                                                                 ui32ClientTAFenceCount,
2897                                                                                 apsClientTAFenceSyncPrimBlock,
2898                                                                                 paui32ClientTAFenceSyncOffset);
2899         if(eError != PVRSRV_OK)
2900         {
2901                 goto err_populate_sync_addr_list;
2902         }
2903
2904         pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
2905
2906         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
2907                                                                                 ui32ClientTAUpdateCount,
2908                                                                                 apsClientTAUpdateSyncPrimBlock,
2909                                                                                 paui32ClientTAUpdateSyncOffset);
2910         if(eError != PVRSRV_OK)
2911         {
2912                 goto err_populate_sync_addr_list;
2913         }
2914
2915         pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
2916
2917         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
2918                                                                                 ui32Client3DFenceCount,
2919                                                                                 apsClient3DFenceSyncPrimBlock,
2920                                                                                 paui32Client3DFenceSyncOffset);
2921         if(eError != PVRSRV_OK)
2922         {
2923                 goto err_populate_sync_addr_list;
2924         }
2925
2926         pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
2927
2928         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
2929                                                                                 ui32Client3DUpdateCount,
2930                                                                                 apsClient3DUpdateSyncPrimBlock,
2931                                                                                 paui32Client3DUpdateSyncOffset);
2932         if(eError != PVRSRV_OK)
2933         {
2934                 goto err_populate_sync_addr_list;
2935         }
2936
2937         pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
2938
2939         eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock,
2940                                                                         ui32PRFenceSyncOffset,
2941                                                                         &uiPRFenceUFOAddress);
2942
2943         if(eError != PVRSRV_OK)
2944         {
2945                 goto err_pr_fence_address;
2946         }
2947
2948
2949
2950         /* Sanity check the server fences */
2951         for (i=0;i<ui32ServerTASyncPrims;i++)
2952         {
2953                 if (!(paui32ServerTASyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK))
2954                 {
2955                         PVR_DPF((PVR_DBG_ERROR, "%s: Server fence (on TA) must fence", __FUNCTION__));
2956                         return PVRSRV_ERROR_INVALID_SYNC_PRIM_OP;
2957                 }
2958         }
2959
2960         for (i=0;i<ui32Server3DSyncPrims;i++)
2961         {
2962                 if (!(paui32Server3DSyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK))
2963                 {
2964                         PVR_DPF((PVR_DBG_ERROR, "%s: Server fence (on 3D) must fence", __FUNCTION__));
2965                         return PVRSRV_ERROR_INVALID_SYNC_PRIM_OP;
2966                 }
2967         }
2968
2969         RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
2970                                   & pPreAddr,
2971                                   & pPostAddr,
2972                                   & pRMWUFOAddr);
2973
2974         /*
2975                 Sanity check we have a PR kick if there are client or server fences
2976         */
2977         if (!bKickPR && ((ui32Client3DFenceCount != 0) || (ui32Server3DSyncPrims != 0)))
2978         {
2979                 PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence (client or server) passed without a PR kick", __FUNCTION__));
2980                 return PVRSRV_ERROR_INVALID_PARAMS;
2981         }
2982
2983         if (ui32SyncPMRCount)
2984         {
2985 #if defined(SUPPORT_BUFFER_SYNC)
2986                 PVRSRV_DEVICE_NODE *psDeviceNode = psRenderContext->psDeviceNode;
2987                 IMG_UINT32 ui32ClientIntUpdateCount = 0;
2988                 PRGXFWIF_UFO_ADDR *pauiClientIntUpdateUFOAddress = NULL;
2989                 IMG_UINT32 *paui32ClientIntUpdateValue = NULL;
2990                 int err;
2991
2992                 if (!bKickTA)
2993                 {
2994                         PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync only supported for kicks including a TA",
2995                                          __FUNCTION__));
2996                         return PVRSRV_ERROR_INVALID_PARAMS;
2997                 }
2998
2999                 if (!bKickPR)
3000                 {
3001                         PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync only supported for kicks including a PR",
3002                                          __FUNCTION__));
3003                         return PVRSRV_ERROR_INVALID_PARAMS;
3004                 }
3005
3006                 if (bKick3D)
3007                 {
3008                         ui32ClientIntUpdateCount = ui32Client3DUpdateCount;
3009                         pauiClientIntUpdateUFOAddress = pauiClient3DUpdateUFOAddress;
3010                         paui32ClientIntUpdateValue = paui32Client3DUpdateValue;
3011                 }
3012                 else
3013                 {
3014                         ui32ClientIntUpdateCount = ui32ClientPRUpdateCount;
3015                         pauiClientIntUpdateUFOAddress = pauiClientPRUpdateUFOAddress;
3016                         paui32ClientIntUpdateValue = paui32ClientPRUpdateValue;
3017                 }
3018
3019                 err = pvr_buffer_sync_append_start(psDeviceNode->psBufferSyncContext,
3020                                                                                    ui32SyncPMRCount,
3021                                                                                    ppsSyncPMRs,
3022                                                                                    paui32SyncPMRFlags,
3023                                                                                    ui32ClientTAFenceCount,
3024                                                                                    pauiClientTAFenceUFOAddress,
3025                                                                                    paui32ClientTAFenceValue,
3026                                                                                    ui32ClientIntUpdateCount,
3027                                                                                    pauiClientIntUpdateUFOAddress,
3028                                                                                    paui32ClientIntUpdateValue,
3029                                                                                    &psAppendData);
3030                 if (err)
3031                 {
3032                         eError = (err == -ENOMEM) ? PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_ERROR_INVALID_PARAMS;
3033                         goto fail_sync_append;
3034                 }
3035
3036                 pvr_buffer_sync_append_checks_get(psAppendData,
3037                                                                                   &ui32ClientTAFenceCount,
3038                                                                                   &pauiClientTAFenceUFOAddress,
3039                                                                                   &paui32ClientTAFenceValue);
3040
3041                 if (bKick3D)
3042                 {
3043                         pvr_buffer_sync_append_updates_get(psAppendData,
3044                                                                                            &ui32Client3DUpdateCount,
3045                                                                                            &pauiClient3DUpdateUFOAddress,
3046                                                                                            &paui32Client3DUpdateValue);
3047                 }
3048                 else
3049                 {
3050                         pvr_buffer_sync_append_updates_get(psAppendData,
3051                                                                                            &ui32ClientPRUpdateCount,
3052                                                                                            &pauiClientPRUpdateUFOAddress,
3053                                                                                            &paui32ClientPRUpdateValue);
3054                 }
3055 #else
3056                 PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __FUNCTION__, ui32SyncPMRCount));
3057                 return PVRSRV_ERROR_INVALID_PARAMS;
3058 #endif /* defined(SUPPORT_BUFFER_SYNC) */
3059         }
3060
3061 #if defined(SUPPORT_NATIVE_FENCE_SYNC)
3062         /*
3063          * The hardware requires a PR to be submitted if there is a TA (otherwise
3064          * it can wedge if we run out of PB space with no PR to run)
3065          *
3066          * If we only have a TA, attach native checks to the TA and updates to the PR
3067          * If we have a TA and 3D, attach checks to TA, updates to 3D
3068          * If we only have a 3D, attach checks and updates to the 3D
3069          *
3070          * Note that 'updates' includes the cleanup syncs for 'check' fence FDs, in
3071          * addition to the update fence FD (if supplied)
3072          *
3073          * Currently, the client driver never kicks only the 3D, so we only support
3074          * that for the time being.
3075          */
3076         if (i32CheckFenceFD >= 0 || i32UpdateTimelineFD >= 0)
3077         {
3078                 IMG_UINT32                      ui32ClientIntUpdateCount = 0;
3079                 PRGXFWIF_UFO_ADDR       *pauiClientIntUpdateUFOAddress = NULL;
3080                 IMG_UINT32                      *paui32ClientIntUpdateValue = NULL;
3081
3082                 if (!bKickTA)
3083                 {
3084                         PVR_DPF((PVR_DBG_ERROR, "%s: Native syncs only supported for kicks including a TA",
3085                                 __FUNCTION__));
3086                         eError = PVRSRV_ERROR_INVALID_PARAMS;
3087                         goto fail_fdsync;
3088                 }
3089                 if (!bKickPR)
3090                 {
3091                         PVR_DPF((PVR_DBG_ERROR, "%s: Native syncs require a PR for all kicks",
3092                                 __FUNCTION__));
3093                         eError = PVRSRV_ERROR_INVALID_PARAMS;
3094                         goto fail_fdsync;
3095                 }
3096                 /* If we have a 3D, attach updates to that. Otherwise, we attach it to a PR */
3097                 if (bKick3D)
3098                 {
3099                         ui32ClientIntUpdateCount = ui32Client3DUpdateCount;
3100                         pauiClientIntUpdateUFOAddress = pauiClient3DUpdateUFOAddress;
3101                         paui32ClientIntUpdateValue = paui32Client3DUpdateValue;
3102                 }
3103                 else
3104                 {
3105                         ui32ClientIntUpdateCount = ui32ClientPRUpdateCount;
3106                         pauiClientIntUpdateUFOAddress = pauiClientPRUpdateUFOAddress;
3107                         paui32ClientIntUpdateValue = paui32ClientPRUpdateValue;
3108                 }
3109
3110                 eError =
3111                         pvr_sync_append_fences(szFenceName,
3112                                                i32CheckFenceFD,
3113                                                i32UpdateTimelineFD,
3114                                                ui32ClientIntUpdateCount,
3115                                                pauiClientIntUpdateUFOAddress,
3116                                                paui32ClientIntUpdateValue,
3117                                                ui32ClientTAFenceCount,
3118                                                pauiClientTAFenceUFOAddress,
3119                                                paui32ClientTAFenceValue,
3120                                                &psFDData);
3121                 if (eError != PVRSRV_OK)
3122                 {
3123                         goto fail_fdsync;
3124                 }
3125                 /* If we have a 3D, attach updates to that. Otherwise, we attach it to a PR */
3126                 if (bKick3D)
3127                 {
3128                         pvr_sync_get_updates(psFDData, &ui32Client3DUpdateCount,
3129                                 &pauiClient3DUpdateUFOAddress, &paui32Client3DUpdateValue);
3130                 }
3131                 else
3132                 {
3133                         pvr_sync_get_updates(psFDData, &ui32ClientPRUpdateCount,
3134                                 &pauiClientPRUpdateUFOAddress, &paui32ClientPRUpdateValue);
3135                 }
3136                 pvr_sync_get_checks(psFDData, &ui32ClientTAFenceCount,
3137                         &pauiClientTAFenceUFOAddress, &paui32ClientTAFenceValue);
3138                 if (ui32ClientPRUpdateCount)
3139                 {
3140                         PVR_ASSERT(pauiClientPRUpdateUFOAddress);
3141                         PVR_ASSERT(paui32ClientPRUpdateValue);
3142                 }
3143                 if (ui32Client3DUpdateCount)
3144                 {
3145                         PVR_ASSERT(pauiClient3DUpdateUFOAddress);
3146                         PVR_ASSERT(paui32Client3DUpdateValue);
3147                 }
3148         }
3149 #endif /* SUPPORT_NATIVE_FENCE_SYNC */
3150
3151         /* Init and acquire to TA command if required */
3152         if(bKickTA)
3153         {
3154                 RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
3155
3156 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3157                 /* Prepare workload estimation */
3158                 WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
3159                                &(psRenderContext->sWorkEstData),
3160                                &(psRenderContext->sWorkEstData.sWorkloadMatchingDataTA),
3161                                ui32RenderTargetSize,
3162                                ui32NumberOfDrawCalls,
3163                                ui32NumberOfIndices,
3164                                ui32NumberOfMRTs,
3165                                ui64DeadlineInus,
3166                                &sWorkloadKickDataTA);
3167 #endif
3168
3169                 /* Init the TA command helper */
3170                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
3171                                                 ui32ClientTAFenceCount,
3172                                                 pauiClientTAFenceUFOAddress,
3173                                                 paui32ClientTAFenceValue,
3174                                                 ui32ClientTAUpdateCount,
3175                                                 pauiClientTAUpdateUFOAddress,
3176                                                 paui32ClientTAUpdateValue,
3177                                                 ui32ServerTASyncPrims,
3178                                                 paui32ServerTASyncFlags,
3179                                                 SYNC_FLAG_MASK_ALL,
3180                                                 pasServerTASyncs,
3181                                                 ui32TACmdSize,
3182                                                 pui8TADMCmd,
3183                                                 & pPreAddr,
3184                                                 (bKick3D ? NULL : & pPostAddr),
3185                                                 (bKick3D ? NULL : & pRMWUFOAddr),
3186                                                 RGXFWIF_CCB_CMD_TYPE_TA,
3187                                                 ui32ExtJobRef,
3188                                                 ui32JobId,
3189                                                 ui32PDumpFlags,
3190 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3191                                                 &sWorkloadKickDataTA,
3192 #else
3193                                                                                 NULL,
3194 #endif
3195                                                 "TA",
3196                                                 asTACmdHelperData);
3197                 if (eError != PVRSRV_OK)
3198                 {
3199                         goto fail_tacmdinit;
3200                 }
3201
3202 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3203                 /* The following is used to determine the offset of the command header
3204                  * containing the workload estimation data so that can be accessed when
3205                  * the KCCB is read.
3206                  */
3207                 ui32TACmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(asTACmdHelperData);
3208 #endif
3209
3210                 eError = RGXCmdHelperAcquireCmdCCB(IMG_ARR_NUM_ELEMS(asTACmdHelperData),
3211                                                    asTACmdHelperData);
3212                 if (eError != PVRSRV_OK)
3213                 {
3214                         goto fail_taacquirecmd;
3215                 }
3216                 else
3217                 {
3218                         ui32TACmdCount++;
3219                 }
3220         }
3221
3222         /* Only kick the 3D if required */
3223         if (bKickPR)
3224         {
3225                 RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
3226
3227                 /*
3228                         The command helper doesn't know about the PR fence so create
3229                         the command with all the fences against it and later create
3230                         the PR command itself which _must_ come after the PR fence.
3231                 */
3232                 sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
3233                 sPRUFO.ui32Value = ui32PRFenceValue;
3234
3235                 /* Init the PR fence command helper */
3236                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
3237                                                                                 ui32Client3DFenceCount,
3238                                                                                 pauiClient3DFenceUFOAddress,
3239                                                                                 paui32Client3DFenceValue,
3240                                                                                 0,
3241                                                                                 NULL,
3242                                                                                 NULL,
3243                                                                                 (bKick3D ? ui32Server3DSyncPrims : 0),
3244                                                                                 paui32Server3DSyncFlags,
3245                                                                                 PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK,
3246                                                                                 pasServer3DSyncs,
3247                                                                                 sizeof(sPRUFO),
3248                                                                                 (IMG_UINT8*) &sPRUFO,
3249                                                                                 NULL,
3250                                                                                 NULL,
3251                                                                                 NULL,
3252                                                                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
3253                                                                                 ui32ExtJobRef,
3254                                                                                 ui32JobId,
3255                                                                                 ui32PDumpFlags,
3256                                                                                 NULL,
3257                                                                                 "3D-PR-Fence",
3258                                                                                 &as3DCmdHelperData[ui323DCmdCount++]);
3259                 if (eError != PVRSRV_OK)
3260                 {
3261                         goto fail_prfencecmdinit;
3262                 }
3263
3264                 /* Init the 3D PR command helper */
3265                 /*
3266                         See note above PVRFDSyncQueryFencesKM as to why updates for android
3267                         syncs are passed in with the PR
3268                 */
3269                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
3270                                                                                 0,
3271                                                                                 NULL,
3272                                                                                 NULL,
3273                                                                                 ui32ClientPRUpdateCount,
3274                                                                                 pauiClientPRUpdateUFOAddress,
3275                                                                                 paui32ClientPRUpdateValue,
3276                                                                                 0,
3277                                                                                 NULL,
3278                                                                                 SYNC_FLAG_MASK_ALL,
3279                                                                                 NULL,
3280                                                                                 ui323DPRCmdSize,
3281                                                                                 pui83DPRDMCmd,
3282                                                                                 NULL,
3283                                                                                 NULL,
3284                                                                                 NULL,
3285                                                                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
3286                                                                                 ui32ExtJobRef,
3287                                                                                 ui32JobId,
3288                                                                                 ui32PDumpFlags,
3289                                                                                 NULL,
3290                                                                                 "3D-PR",
3291                                                                                 &as3DCmdHelperData[ui323DCmdCount++]);
3292                 if (eError != PVRSRV_OK)
3293                 {
3294                         goto fail_prcmdinit;
3295                 }
3296         }
3297
3298         if (bKick3D || bAbort)
3299         {
3300                 RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
3301
3302 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3303                 /* Prepare workload estimation */
3304                 WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
3305                                &(psRenderContext->sWorkEstData),
3306                                &(psRenderContext->sWorkEstData.sWorkloadMatchingData3D),
3307                                ui32RenderTargetSize,
3308                                ui32NumberOfDrawCalls,
3309                                ui32NumberOfIndices,
3310                                ui32NumberOfMRTs,
3311                                ui64DeadlineInus,
3312                                &sWorkloadKickData3D);
3313 #endif
3314                 /* Init the 3D command helper */
3315                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
3316                                                                                 0,
3317                                                                                 NULL,
3318                                                                                 NULL,
3319                                                                                 ui32Client3DUpdateCount,
3320                                                                                 pauiClient3DUpdateUFOAddress,
3321                                                                                 paui32Client3DUpdateValue,
3322                                                                                 ui32Server3DSyncPrims,
3323                                                                                 paui32Server3DSyncFlags,
3324                                                                                 PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE,
3325                                                                                 pasServer3DSyncs,
3326                                                                                 ui323DCmdSize,
3327                                                                                 pui83DDMCmd,
3328                                                                                 (bKickTA ? NULL : & pPreAddr),
3329                                                                                 & pPostAddr,
3330                                                                                 & pRMWUFOAddr,
3331                                                                                 RGXFWIF_CCB_CMD_TYPE_3D,
3332                                                                                 ui32ExtJobRef,
3333                                                                                 ui32JobId,
3334                                                                                 ui32PDumpFlags,
3335 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3336                                                                                 &sWorkloadKickData3D,
3337 #else
3338                                                                                 NULL,
3339 #endif
3340                                                                                 "3D",
3341                                                                                 &as3DCmdHelperData[ui323DCmdCount++]);
3342                 if (eError != PVRSRV_OK)
3343                 {
3344                         goto fail_3dcmdinit;
3345                 }
3346
3347 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3348                 /* The following are used to determine the offset of the command header
3349                  * containing the workload estimation data so that can be accessed when
3350                  * the KCCB is read.
3351                  */
3352                 ui323DCmdHeaderOffset =
3353                         RGXCmdHelperGetDMCommandHeaderOffset(&as3DCmdHelperData[ui323DCmdCount - 1]);
3354                 ui323DFullRenderCommandOffset =
3355                         RGXCmdHelperGetCommandOffset(as3DCmdHelperData,
3356                                                      ui323DCmdCount - 1);
3357 #endif
3358         }
3359
3360         /* Protect against array overflow in RGXCmdHelperAcquireCmdCCB() */
3361         if (ui323DCmdCount > IMG_ARR_NUM_ELEMS(as3DCmdHelperData))
3362         {
3363                 goto fail_3dcmdinit;
3364         }
3365
3366         if (ui323DCmdCount)
3367         {
3368                 PVR_ASSERT(bKickPR || bKick3D);
3369
3370                 /* Acquire space for all the 3D command(s) */
3371                 eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount,
3372                                                                                    as3DCmdHelperData);
3373                 if (eError != PVRSRV_OK)
3374                 {
3375                         /* If RGXCmdHelperAcquireCmdCCB fails we skip the scheduling
3376                          * of a new TA command with the same Write offset in Kernel CCB.
3377                          */
3378                         goto fail_3dacquirecmd;
3379                 }
3380         }
3381
3382         /*
3383                 We should acquire the space in the kernel CCB here as after this point
3384                 we release the commands which will take operations on server syncs
3385                 which can't be undone
3386         */
3387
3388         /*
3389                 Everything is ready to go now, release the commands
3390         */
3391         if (ui32TACmdCount)
3392         {
3393                 ui32TACmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
3394                 RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
3395                                                                   asTACmdHelperData,
3396                                                                   "TA",
3397                                                                   FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
3398
3399 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3400                 ui32TACmdOffsetWrapCheck =
3401                         RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
3402
3403                 /* This checks if the command would wrap around at the end of the CCB
3404                  * and therefore would start at an offset of 0 rather than the current
3405                  * command offset.
3406                  */
3407                 if(ui32TACmdOffset < ui32TACmdOffsetWrapCheck)
3408                 {
3409                         ui32TACommandOffset = ui32TACmdOffset;
3410                 }
3411                 else
3412                 {
3413                         ui32TACommandOffset = 0;
3414                 }
3415 #endif
3416         }
3417
3418         if (ui323DCmdCount)
3419         {
3420                 ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
3421                 RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
3422                                                                   as3DCmdHelperData,
3423                                                                   "3D",
3424                                                                   FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
3425 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3426                 ui323DCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
3427
3428                 if(ui323DCmdOffset < ui323DCmdOffsetWrapCheck)
3429                 {
3430                         ui323DCommandOffset = ui323DCmdOffset;
3431                 }
3432                 else
3433                 {
3434                         ui323DCommandOffset = 0;
3435                 }
3436 #endif
3437         }
3438
3439         if (ui32TACmdCount)
3440         {
3441                 RGXFWIF_KCCB_CMD sTAKCCBCmd;
3442                 IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr;
3443
3444                 /* Construct the kernel TA CCB command. */
3445                 sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
3446                 sTAKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
3447                 sTAKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
3448
3449                 /* Add the Workload data into the KCCB kick */
3450                 sTAKCCBCmd.uCmdData.sCmdKickData.sWorkloadDataFWAddress.ui32Addr = 0;
3451                 sTAKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
3452
3453 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3454                 /* Store the offset to the CCCB command header so that it can be
3455                  * referenced when the KCCB command reaches the FW
3456                  */
3457                 sTAKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset =
3458                         ui32TACommandOffset + ui32TACmdHeaderOffset;
3459 #endif
3460
3461                 if(bCommitRefCountsTA)
3462                 {
3463                         AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTAKCCBCmd.uCmdData.sCmdKickData.apsCleanupCtl,
3464                                                                                 &sTAKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl,
3465                                                                                 RGXFWIF_DM_TA,
3466                                                                                 bKickTA,
3467                                                                                 psRTDataCleanup,
3468                                                                                 psZBuffer,
3469                                                                                 psSBuffer);
3470                         *pbCommittedRefCountsTA = IMG_TRUE;
3471                 }
3472                 else
3473                 {
3474                         sTAKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
3475                 }
3476
3477                 HTBLOGK(HTB_SF_MAIN_KICK_TA,
3478                                 sTAKCCBCmd.uCmdData.sCmdKickData.psContext,
3479                                 ui32TACmdOffset
3480                                 );
3481                 RGX_HWPERF_HOST_ENQ(psRenderContext, OSGetCurrentClientProcessIDKM(),
3482                                     ui32FWCtx, ui32ExtJobRef, ui32JobId,
3483                                     RGX_HWPERF_KICK_TYPE_TA3D);
3484
3485                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3486                 {
3487                         eError2 = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
3488                                                                                 RGXFWIF_DM_TA,
3489                                                                                 &sTAKCCBCmd,
3490                                                                                 sizeof(sTAKCCBCmd),
3491                                                                                 ui32ClientCacheOpSeqNum,
3492                                                                                 ui32PDumpFlags);
3493                         if (eError2 != PVRSRV_ERROR_RETRY)
3494                         {
3495                                 break;
3496                         }
3497                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3498                 } END_LOOP_UNTIL_TIMEOUT();
3499
3500 #if defined(SUPPORT_GPUTRACE_EVENTS)
3501                 RGXHWPerfFTraceGPUEnqueueEvent(psRenderContext->psDeviceNode->pvDevice,
3502                                         ui32FWCtx, ui32JobId, RGX_HWPERF_KICK_TYPE_TA3D);
3503 #endif
3504         }
3505
3506         if (ui323DCmdCount)
3507         {
3508                 RGXFWIF_KCCB_CMD s3DKCCBCmd;
3509
3510                 /* Construct the kernel 3D CCB command. */
3511                 s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
3512                 s3DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
3513                 s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
3514
3515                 /* Add the Workload data into the KCCB kick */
3516                 s3DKCCBCmd.uCmdData.sCmdKickData.sWorkloadDataFWAddress.ui32Addr = 0;
3517                 s3DKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
3518
3519 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
3520                 /* Store the offset to the CCCB command header so that it can be
3521                  * referenced when the KCCB command reaches the FW
3522                  */
3523                 s3DKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui323DCommandOffset + ui323DCmdHeaderOffset + ui323DFullRenderCommandOffset;
3524 #endif
3525
3526                 if(bCommitRefCounts3D)
3527                 {
3528                         AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DKCCBCmd.uCmdData.sCmdKickData.apsCleanupCtl,
3529                                                                                         &s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl,
3530                                                                                         RGXFWIF_DM_3D,
3531                                                                                         bKick3D,
3532                                                                                         psRTDataCleanup,
3533                                                                                         psZBuffer,
3534                                                                                         psSBuffer);
3535                         *pbCommittedRefCounts3D = IMG_TRUE;
3536                 }
3537                 else
3538                 {
3539                         s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
3540                 }
3541
3542
3543                 HTBLOGK(HTB_SF_MAIN_KICK_3D,
3544                                 s3DKCCBCmd.uCmdData.sCmdKickData.psContext,
3545                                 ui323DCmdOffset);
3546
3547                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3548                 {
3549                         eError2 = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
3550                                                                                 RGXFWIF_DM_3D,
3551                                                                                 &s3DKCCBCmd,
3552                                                                                 sizeof(s3DKCCBCmd),
3553                                                                                 ui32ClientCacheOpSeqNum,
3554                                                                                 ui32PDumpFlags);
3555                         if (eError2 != PVRSRV_ERROR_RETRY)
3556                         {
3557                                 break;
3558                         }
3559                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3560                 } END_LOOP_UNTIL_TIMEOUT();
3561         }
3562
3563         /*
3564          * Now check eError (which may have returned an error from our earlier calls
3565          * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
3566          * so we check it now...
3567          */
3568         if (eError != PVRSRV_OK )
3569         {
3570                 goto fail_3dacquirecmd;
3571         }
3572
3573 #if defined(SUPPORT_NATIVE_FENCE_SYNC)
3574         if (i32UpdateTimelineFD >= 0)
3575         {
3576                 /* If we get here, this should never fail. Hitting that likely implies
3577                  * a code error above */
3578                 i32UpdateFenceFD = pvr_sync_get_update_fd(psFDData);
3579                 if (i32UpdateFenceFD < 0)
3580                 {
3581                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get install update sync fd",
3582                                 __FUNCTION__));
3583                         /* If we fail here, we cannot rollback the syncs as the hw already
3584                          * has references to resources they may be protecting in the kick
3585                          * so fallthrough */
3586
3587                         eError = PVRSRV_ERROR_INVALID_PARAMS;
3588                         goto fail_3dacquirecmd;
3589                 }
3590         }
3591 #if defined(NO_HARDWARE)
3592         pvr_sync_nohw_complete_fences(psFDData);
3593 #endif
3594         pvr_sync_free_append_fences_data(psFDData);
3595
3596 #endif
3597
3598 #if defined(SUPPORT_BUFFER_SYNC)
3599         if (psAppendData)
3600         {
3601                 pvr_buffer_sync_append_finish(psAppendData);
3602         }
3603 #endif
3604
3605         *pi32UpdateFenceFD = i32UpdateFenceFD;
3606
3607         return PVRSRV_OK;
3608
3609 fail_3dacquirecmd:
3610 fail_3dcmdinit:
3611 fail_prcmdinit:
3612 fail_prfencecmdinit:
3613 fail_taacquirecmd:
3614 fail_tacmdinit:
3615 #if defined(SUPPORT_NATIVE_FENCE_SYNC)
3616         pvr_sync_rollback_append_fences(psFDData);
3617         pvr_sync_free_append_fences_data(psFDData);
3618 fail_fdsync:
3619 #endif
3620 #if defined(SUPPORT_BUFFER_SYNC)
3621         pvr_buffer_sync_append_abort(psAppendData);
3622 fail_sync_append:
3623 #endif
3624 err_pr_fence_address:
3625 err_populate_sync_addr_list:
3626         PVR_ASSERT(eError != PVRSRV_OK);
3627         return eError;
3628 }
3629
3630 PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
3631                                                  PVRSRV_DEVICE_NODE * psDeviceNode,
3632                                                                                                  RGX_SERVER_RENDER_CONTEXT *psRenderContext,
3633                                                                                                  IMG_UINT32 ui32Priority)
3634 {
3635         PVRSRV_ERROR eError;
3636
3637         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
3638
3639         if (psRenderContext->sTAData.ui32Priority != ui32Priority)
3640         {
3641                 eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
3642                                                                         psConnection,
3643                                                                         psRenderContext->psDeviceNode->pvDevice,
3644                                                                         ui32Priority,
3645                                                                         RGXFWIF_DM_TA);
3646                 if (eError != PVRSRV_OK)
3647                 {
3648                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the TA part of the rendercontext (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
3649                         goto fail_tacontext;
3650                 }
3651                 psRenderContext->sTAData.ui32Priority = ui32Priority;
3652         }
3653
3654         if (psRenderContext->s3DData.ui32Priority != ui32Priority)
3655         {
3656                 eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
3657                                                                         psConnection,
3658                                                                         psRenderContext->psDeviceNode->pvDevice,
3659                                                                         ui32Priority,
3660                                                                         RGXFWIF_DM_3D);
3661                 if (eError != PVRSRV_OK)
3662                 {
3663                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 3D part of the rendercontext (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
3664                         goto fail_3dcontext;
3665                 }
3666                 psRenderContext->s3DData.ui32Priority = ui32Priority;
3667         }
3668         return PVRSRV_OK;
3669
3670 fail_3dcontext:
3671 fail_tacontext:
3672         PVR_ASSERT(eError != PVRSRV_OK);
3673         return eError;
3674 }
3675
3676
3677 /*
3678  * PVRSRVRGXGetLastRenderContextResetReasonKM
3679  */
3680 PVRSRV_ERROR PVRSRVRGXGetLastRenderContextResetReasonKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
3681                                                         IMG_UINT32 *peLastResetReason,
3682                                                         IMG_UINT32 *pui32LastResetJobRef)
3683 {
3684         RGX_SERVER_RC_TA_DATA         *psRenderCtxTAData;
3685         RGX_SERVER_RC_3D_DATA         *psRenderCtx3DData;
3686         RGX_SERVER_COMMON_CONTEXT     *psCurrentServerTACommonCtx, *psCurrentServer3DCommonCtx;
3687         RGXFWIF_CONTEXT_RESET_REASON  eLastTAResetReason, eLast3DResetReason;
3688         IMG_UINT32                    ui32LastTAResetJobRef, ui32Last3DResetJobRef;
3689
3690         PVR_ASSERT(psRenderContext != NULL);
3691         PVR_ASSERT(peLastResetReason != NULL);
3692         PVR_ASSERT(pui32LastResetJobRef != NULL);
3693
3694         psRenderCtxTAData          = &(psRenderContext->sTAData);
3695         psCurrentServerTACommonCtx = psRenderCtxTAData->psServerCommonContext;
3696         psRenderCtx3DData          = &(psRenderContext->s3DData);
3697         psCurrentServer3DCommonCtx = psRenderCtx3DData->psServerCommonContext;
3698
3699         /* Get the last reset reasons from both the TA and 3D so they are reset... */
3700         eLastTAResetReason = FWCommonContextGetLastResetReason(psCurrentServerTACommonCtx, &ui32LastTAResetJobRef);
3701         eLast3DResetReason = FWCommonContextGetLastResetReason(psCurrentServer3DCommonCtx, &ui32Last3DResetJobRef);
3702
3703         /* Combine the reset reason from TA and 3D into one... */
3704         *peLastResetReason    = (IMG_UINT32) eLast3DResetReason;
3705         *pui32LastResetJobRef = ui32Last3DResetJobRef;
3706         if (eLast3DResetReason == RGXFWIF_CONTEXT_RESET_REASON_NONE  ||
3707             ((eLast3DResetReason == RGXFWIF_CONTEXT_RESET_REASON_INNOCENT_LOCKUP  ||
3708               eLast3DResetReason == RGXFWIF_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING)  &&
3709              (eLastTAResetReason == RGXFWIF_CONTEXT_RESET_REASON_GUILTY_LOCKUP  ||
3710               eLastTAResetReason == RGXFWIF_CONTEXT_RESET_REASON_GUILTY_OVERRUNING)))
3711         {
3712                 *peLastResetReason    = eLastTAResetReason;
3713                 *pui32LastResetJobRef = ui32LastTAResetJobRef;
3714         }
3715
3716         return PVRSRV_OK;
3717 }
3718
3719
3720 /*
3721  * PVRSRVRGXGetPartialRenderCountKM
3722  */
3723 PVRSRV_ERROR PVRSRVRGXGetPartialRenderCountKM(DEVMEM_MEMDESC *psHWRTDataMemDesc,
3724                                                                                           IMG_UINT32 *pui32NumPartialRenders)
3725 {
3726         RGXFWIF_HWRTDATA *psHWRTData;
3727         PVRSRV_ERROR eError;
3728
3729         eError = DevmemAcquireCpuVirtAddr(psHWRTDataMemDesc, (void **)&psHWRTData);
3730
3731         if (eError != PVRSRV_OK)
3732         {
3733                 PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXGetPartialRenderCountKM: Failed to map Firmware Render Target Data (%u)", eError));
3734                 return eError;
3735         }
3736
3737         *pui32NumPartialRenders = psHWRTData->ui32NumPartialRenders;
3738
3739         DevmemReleaseCpuVirtAddr(psHWRTDataMemDesc);
3740
3741         return PVRSRV_OK;
3742 }
3743
3744 void CheckForStalledRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo,
3745                                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
3746                                         void *pvDumpDebugFile)
3747 {
3748         DLLIST_NODE *psNode, *psNext;
3749         OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
3750         dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
3751         {
3752                 RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
3753                         IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
3754
3755                 DumpStalledFWCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext,
3756                                                                    pfnDumpDebugPrintf, pvDumpDebugFile);
3757                 DumpStalledFWCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext,
3758                                                                    pfnDumpDebugPrintf, pvDumpDebugFile);
3759         }
3760         OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
3761 }
3762
3763 IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
3764 {
3765         DLLIST_NODE *psNode, *psNext;
3766         IMG_UINT32 ui32ContextBitMask = 0;
3767
3768         OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
3769
3770         dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
3771         {
3772                 RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
3773                         IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
3774                 if(NULL != psCurrentServerRenderCtx->sTAData.psServerCommonContext)
3775                 {
3776                         if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext, RGX_KICK_TYPE_DM_TA) == PVRSRV_ERROR_CCCB_STALLED)
3777                         {
3778                                 ui32ContextBitMask |= RGX_KICK_TYPE_DM_TA;
3779                         }
3780                 }
3781
3782                 if(NULL != psCurrentServerRenderCtx->s3DData.psServerCommonContext)
3783                 {
3784                         if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_3D) == PVRSRV_ERROR_CCCB_STALLED)
3785                         {
3786                                 ui32ContextBitMask |= RGX_KICK_TYPE_DM_3D;
3787                         }
3788                 }
3789         }
3790
3791         OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
3792         return ui32ContextBitMask;
3793 }
3794
3795 /******************************************************************************
3796  End of file (rgxta3d.c)
3797 ******************************************************************************/