drivers/staging/imgtec/rogue/rgxfwutils.c

   1  /*************************************************************************/ /*!
   2 @File
   3 @Title          Rogue firmware utility routines
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Rogue firmware utility routines
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43
  44 #include <stddef.h>
  45
  46 #include "lists.h"
  47
  48 #include "rgxdefs_km.h"
  49 #include "rgx_fwif_km.h"
  50 #include "pdump_km.h"
  51 #include "osfunc.h"
  52 #include "cache_km.h"
  53 #include "allocmem.h"
  54 #include "devicemem.h"
  55 #include "devicemem_pdump.h"
  56 #include "devicemem_server.h"
  57
  58 #include "pvr_debug.h"
  59 #include "pvr_notifier.h"
  60 #include "rgxfwutils.h"
  61 #include "rgx_options.h"
  62 #include "rgx_fwif.h"
  63 #include "rgx_fwif_alignchecks.h"
  64 #include "rgx_fwif_resetframework.h"
  65 #include "rgx_pdump_panics.h"
  66 #include "rgxheapconfig.h"
  67 #include "pvrsrv.h"
  68 #if defined(SUPPORT_PVRSRV_GPUVIRT)
  69 #include "rgxfwutils_vz.h"
  70 #endif
  71 #include "rgxdebug.h"
  72 #include "rgxhwperf.h"
  73 #include "rgxccb.h"
  74 #include "rgxcompute.h"
  75 #include "rgxtransfer.h"
  76 #include "rgxpower.h"
  77 #include "rgxray.h"
  78 #if defined(SUPPORT_DISPLAY_CLASS)
  79 #include "dc_server.h"
  80 #endif
  81 #include "rgxmem.h"
  82 #include "rgxta3d.h"
  83 #include "rgxutils.h"
  84 #include "sync_internal.h"
  85 #include "sync.h"
  86 #include "tlstream.h"
  87 #include "devicemem_server_utils.h"
  88 #include "htbuffer.h"
  89 #include "rgx_bvnc_defs_km.h"
  90
  91 #if defined(SUPPORT_TRUSTED_DEVICE)
  92 #include "physmem_osmem.h"
  93 #endif
  94
  95 #ifdef __linux__
  96 #include <linux/kernel.h>    // sprintf
  97 #include <linux/string.h>    // strncpy, strlen
  98 #include "rogue_trace_events.h"
  99 #else
 100 #include <stdio.h>
 101 #include <string.h>
 102 #endif
 103 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
 104 #include "process_stats.h"
 105 #endif
 106
 107 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
 108 #include "rgxworkest.h"
 109 #endif
 110
 111 #if defined(SUPPORT_PDVFS)
 112 #include "rgxpdvfs.h"
 113 #endif
 114
 115 /* Kernel CCB length */
 116         /* Reducing the size of the KCCB in an attempt to avoid flooding and overflowing the FW kick queue
 117          * in the case of multiple OSes */
 118 #define RGXFWIF_KCCB_NUMCMDS_LOG2_GPUVIRT_ONLY    (6)
 119 #define RGXFWIF_KCCB_NUMCMDS_LOG2_FEAT_GPU_VIRTUALISATION    (7)
 120
 121
 122 /* Firmware CCB length */
 123 #if defined(SUPPORT_PDVFS)
 124 #define RGXFWIF_FWCCB_NUMCMDS_LOG2   (8)
 125 #else
 126 #define RGXFWIF_FWCCB_NUMCMDS_LOG2   (5)
 127 #endif
 128
 129 /* Workload Estimation Firmware CCB length */
 130 #define RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2   (7)
 131
 132 typedef struct
 133 {
 134         RGXFWIF_KCCB_CMD        sKCCBcmd;
 135         DLLIST_NODE             sListNode;
 136         PDUMP_FLAGS_T           uiPdumpFlags;
 137         PVRSRV_RGXDEV_INFO      *psDevInfo;
 138         RGXFWIF_DM              eDM;
 139 } RGX_DEFERRED_KCCB_CMD;
 140
 141 #if defined(PDUMP)
 142 /* ensure PIDs are 32-bit because a 32-bit PDump load is generated for the
 143  * PID filter example entries
 144  */
 145 static_assert(sizeof(IMG_PID) == sizeof(IMG_UINT32),
 146                 "FW PID filtering assumes the IMG_PID type is 32-bits wide as it "
 147                 "generates WRW commands for loading the PID values");
 148 #endif
 149
 150 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
 151 static PVRSRV_ERROR _AllocateSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo, RGXFWIF_INIT* psRGXFWInit)
 152 {
 153         PVRSRV_ERROR eError;
 154         DEVMEM_MEMDESC** ppsSLC3FenceMemDesc = &psDevInfo->psSLC3FenceMemDesc;
 155         IMG_UINT32      ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(psDevInfo->sDevFeatureCfg.ui32CacheLineSize);
 156
 157         PVR_DPF_ENTERED;
 158
 159         eError = DevmemAllocate(psDevInfo->psFirmwareHeap,
 160                                                         1,
 161                                                         ui32CacheLineSize,
 162                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 163                             PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 164                                                         PVRSRV_MEMALLOCFLAG_UNCACHED |
 165                                                         PVRSRV_MEMALLOCFLAG_FW_LOCAL,
 166                                                         "SLC3 Fence WA",
 167                                                         ppsSLC3FenceMemDesc);
 168         if (eError != PVRSRV_OK)
 169         {
 170                 PVR_DPF_RETURN_RC(eError);
 171         }
 172
 173         /*
 174                 We need to map it so the heap for this allocation
 175                 is set
 176         */
 177         eError = DevmemMapToDevice(*ppsSLC3FenceMemDesc,
 178                                                            psDevInfo->psFirmwareHeap,
 179                                                            &psRGXFWInit->sSLC3FenceDevVAddr);
 180         if (eError != PVRSRV_OK)
 181         {
 182                 DevmemFwFree(psDevInfo, *ppsSLC3FenceMemDesc);
 183                 *ppsSLC3FenceMemDesc = NULL;
 184         }
 185
 186         PVR_DPF_RETURN_RC1(eError, *ppsSLC3FenceMemDesc);
 187 }
 188
 189 static void _FreeSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo)
 190 {
 191         DEVMEM_MEMDESC* psSLC3FenceMemDesc = psDevInfo->psSLC3FenceMemDesc;
 192
 193         if (psSLC3FenceMemDesc)
 194         {
 195                 DevmemReleaseDevVirtAddr(psSLC3FenceMemDesc);
 196                 DevmemFree(psSLC3FenceMemDesc);
 197         }
 198 }
 199 #endif
 200
 201 static void __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
 202 {
 203         /* ensure memory is flushed before kicking MTS */
 204         OSWriteMemoryBarrier();
 205
 206         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
 207
 208         /* ensure the MTS kick goes through before continuing */
 209         OSMemoryBarrier();
 210 }
 211
 212
 213 /*!
 214 *******************************************************************************
 215  @Function              RGXFWSetupSignatureChecks
 216  @Description
 217  @Input                 psDevInfo
 218
 219  @Return                PVRSRV_ERROR
 220 ******************************************************************************/
 221 static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
 222                                               DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
 223                                               IMG_UINT32          ui32SigChecksBufSize,
 224                                               RGXFWIF_SIGBUF_CTL* psSigBufCtl,
 225                                               const IMG_CHAR*     pszBufferName)
 226 {
 227         PVRSRV_ERROR    eError;
 228         DEVMEM_FLAGS_T  uiMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 229                                                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 230                                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 231                                                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 232                                                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 233                                                                           PVRSRV_MEMALLOCFLAG_UNCACHED |
 234                                                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 235
 236         /* Allocate memory for the checks */
 237         PDUMPCOMMENT("Allocate memory for %s signature checks", pszBufferName);
 238         eError = DevmemFwAllocate(psDevInfo,
 239                                                         ui32SigChecksBufSize,
 240                                                         uiMemAllocFlags,
 241                                                         "FwSignatureChecks",
 242                                                         ppsSigChecksMemDesc);
 243         if (eError != PVRSRV_OK)
 244         {
 245                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %d bytes for signature checks (%u)",
 246                                         ui32SigChecksBufSize,
 247                                         eError));
 248                 return eError;
 249         }
 250
 251         /* Prepare the pointer for the fw to access that memory */
 252         RGXSetFirmwareAddress(&psSigBufCtl->sBuffer,
 253                                                   *ppsSigChecksMemDesc,
 254                                                   0, RFW_FWADDR_NOREF_FLAG);
 255
 256         DevmemPDumpLoadMem(     *ppsSigChecksMemDesc,
 257                                                 0,
 258                                                 ui32SigChecksBufSize,
 259                                                 PDUMP_FLAGS_CONTINUOUS);
 260
 261         psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
 262
 263         return PVRSRV_OK;
 264 }
 265
 266 #if defined(RGXFW_ALIGNCHECKS)
 267 /*!
 268 *******************************************************************************
 269  @Function              RGXFWSetupAlignChecks
 270  @Description   This functions allocates and fills memory needed for the
 271                 aligns checks of the UM and KM structures shared with the
 272                 firmware. The format of the data in the memory is as follows:
 273                     <number of elements in the KM array>
 274                     <array of KM structures' sizes and members' offsets>
 275                     <number of elements in the UM array>
 276                     <array of UM structures' sizes and members' offsets>
 277                 The UM array is passed from the user side. If the
 278                 SUPPORT_KERNEL_SRVINIT macro is defined the firmware is
 279                 is responsible for filling this part of the memory. If that
 280                 happens the check of the UM structures will be performed
 281                 by the host driver on client's connect.
 282                 If the macro is not defined the client driver fills the memory
 283                 and the firmware checks for the alignment of all structures.
 284  @Input                 psDevInfo
 285
 286  @Return                PVRSRV_ERROR
 287 ******************************************************************************/
 288 static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
 289                                                                 RGXFWIF_DEV_VIRTADDR    *psAlignChecksDevFW,
 290                                                                 IMG_UINT32                              *pui32RGXFWAlignChecks,
 291                                                                 IMG_UINT32                              ui32RGXFWAlignChecksArrLength)
 292 {
 293         IMG_UINT32              aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
 294         IMG_UINT32              ui32RGXFWAlingChecksTotal;
 295         IMG_UINT32*             paui32AlignChecks;
 296         PVRSRV_ERROR    eError;
 297
 298 #if defined(SUPPORT_KERNEL_SRVINIT)
 299         /* In this case we don't know the number of elements in UM array.
 300          * We have to assume something so we assume RGXFW_ALIGN_CHECKS_UM_MAX. */
 301         PVR_ASSERT(ui32RGXFWAlignChecksArrLength == 0);
 302         ui32RGXFWAlingChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
 303                 + RGXFW_ALIGN_CHECKS_UM_MAX * sizeof(IMG_UINT32)
 304                 + 2 * sizeof(IMG_UINT32);
 305 #else
 306         /* '2 * sizeof(IMG_UINT32)' if for sizes of km and um arrays. */
 307         PVR_ASSERT(ui32RGXFWAlignChecksArrLength != 0);
 308         ui32RGXFWAlingChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
 309                 + ui32RGXFWAlignChecksArrLength * sizeof(IMG_UINT32)
 310                 + 2 * sizeof(IMG_UINT32);
 311 #endif
 312
 313         /* Allocate memory for the checks */
 314         PDUMPCOMMENT("Allocate memory for alignment checks");
 315         eError = DevmemFwAllocate(psDevInfo,
 316                                                         ui32RGXFWAlingChecksTotal,
 317                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 318                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 319 #if defined(SUPPORT_KERNEL_SRVINIT)
 320                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 321 #endif
 322                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 323                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 324                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | PVRSRV_MEMALLOCFLAG_UNCACHED,
 325                                                         "FwAlignmentChecks",
 326                                                         &psDevInfo->psRGXFWAlignChecksMemDesc);
 327         if (eError != PVRSRV_OK)
 328         {
 329                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %d bytes for alignment checks (%u)",
 330                                         ui32RGXFWAlingChecksTotal,
 331                                         eError));
 332                 goto failAlloc;
 333         }
 334
 335         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
 336                                                                         (void **)&paui32AlignChecks);
 337         if (eError != PVRSRV_OK)
 338         {
 339                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel addr for alignment checks (%u)",
 340                                         eError));
 341                 goto failAqCpuAddr;
 342         }
 343
 344         /* Copy the values */
 345 #if defined(SUPPORT_KERNEL_SRVINIT)
 346         *paui32AlignChecks++ = sizeof(aui32RGXFWAlignChecksKM)/sizeof(IMG_UINT32);
 347         OSDeviceMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0], sizeof(aui32RGXFWAlignChecksKM));
 348         paui32AlignChecks += sizeof(aui32RGXFWAlignChecksKM)/sizeof(IMG_UINT32);
 349
 350         *paui32AlignChecks = 0;
 351 #else
 352         *paui32AlignChecks++ = sizeof(aui32RGXFWAlignChecksKM)/sizeof(IMG_UINT32);
 353         OSDeviceMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0], sizeof(aui32RGXFWAlignChecksKM));
 354         paui32AlignChecks += sizeof(aui32RGXFWAlignChecksKM)/sizeof(IMG_UINT32);
 355
 356         *paui32AlignChecks++ = ui32RGXFWAlignChecksArrLength;
 357         OSDeviceMemCopy(paui32AlignChecks, pui32RGXFWAlignChecks, ui32RGXFWAlignChecksArrLength * sizeof(IMG_UINT32));
 358 #endif
 359
 360         DevmemPDumpLoadMem(     psDevInfo->psRGXFWAlignChecksMemDesc,
 361                                                 0,
 362                                                 ui32RGXFWAlingChecksTotal,
 363                                                 PDUMP_FLAGS_CONTINUOUS);
 364
 365         /* Prepare the pointer for the fw to access that memory */
 366         RGXSetFirmwareAddress(psAlignChecksDevFW,
 367                                                   psDevInfo->psRGXFWAlignChecksMemDesc,
 368                                                   0, RFW_FWADDR_NOREF_FLAG);
 369
 370         return PVRSRV_OK;
 371
 372
 373
 374
 375 failAqCpuAddr:
 376         DevmemFwFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
 377         psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
 378 failAlloc:
 379
 380         PVR_ASSERT(eError != PVRSRV_OK);
 381         return eError;
 382 }
 383
 384 static void RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
 385 {
 386         if (psDevInfo->psRGXFWAlignChecksMemDesc != NULL)
 387         {
 388                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
 389                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
 390                 psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
 391         }
 392 }
 393 #endif
 394
 395
 396 void RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR *ppDest,
 397                                                    DEVMEM_MEMDESC               *psSrc,
 398                                                    IMG_UINT32                   uiExtraOffset,
 399                                                    IMG_UINT32                   ui32Flags)
 400 {
 401         PVRSRV_ERROR            eError;
 402         IMG_DEV_VIRTADDR        psDevVirtAddr;
 403         PVRSRV_DEVICE_NODE      *psDeviceNode;
 404         IMG_UINT64                      ui64ErnsBrns = 0;
 405         PVRSRV_RGXDEV_INFO      *psDevInfo;
 406
 407         psDeviceNode = (PVRSRV_DEVICE_NODE *) DevmemGetConnection(psSrc);
 408         psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
 409         ui64ErnsBrns = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
 410
 411         if(psDevInfo->sDevFeatureCfg.ui32META)
 412         {
 413                 IMG_UINT32          ui32Offset;
 414                 IMG_BOOL            bCachedInMETA;
 415                 DEVMEM_FLAGS_T      uiDevFlags;
 416                 IMG_UINT32          uiGPUCacheMode;
 417
 418                 eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
 419                 PVR_ASSERT(eError == PVRSRV_OK);
 420
 421                 /* Convert to an address in META memmap */
 422                 ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_HEAP_BASE ;
 423
 424                 /* Check in the devmem flags whether this memory is cached/uncached */
 425                 DevmemGetFlags(psSrc, &uiDevFlags);
 426
 427                 /* Honour the META cache flags */
 428                 bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
 429
 430                 /* Honour the SLC cache flags */
 431                 uiGPUCacheMode = DevmemDeviceCacheMode(psDeviceNode, uiDevFlags);
 432
 433                 ui32Offset += RGXFW_SEGMMU_DATA_BASE_ADDRESS;
 434
 435                 if (bCachedInMETA)
 436                 {
 437                         ui32Offset |= RGXFW_SEGMMU_DATA_META_CACHED;
 438                 }
 439                 else
 440                 {
 441                         ui32Offset |= RGXFW_SEGMMU_DATA_META_UNCACHED;
 442                 }
 443
 444                 if (PVRSRV_CHECK_GPU_CACHED(uiGPUCacheMode))
 445                 {
 446                         ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED;
 447                 }
 448                 else
 449                 {
 450                         ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
 451                 }
 452                 ppDest->ui32Addr = ui32Offset;
 453         }else
 454         {
 455                 eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
 456                 PVR_ASSERT(eError == PVRSRV_OK);
 457                 ppDest->ui32Addr = (IMG_UINT32)((psDevVirtAddr.uiAddr + uiExtraOffset) & 0xFFFFFFFF);
 458         }
 459
 460         if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
 461         {
 462                 DevmemReleaseDevVirtAddr(psSrc);
 463         }
 464
 465 }
 466
 467 void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR              *psDest,
 468                                                   DEVMEM_MEMDESC                *psSrcMemDesc,
 469                                                   RGXFWIF_DEV_VIRTADDR  *psSrcFWDevVAddr,
 470                                                   IMG_UINT32                    uiOffset)
 471 {
 472         PVRSRV_ERROR            eError;
 473         IMG_DEV_VIRTADDR        sDevVirtAddr;
 474
 475         eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
 476         PVR_ASSERT(eError == PVRSRV_OK);
 477
 478         psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
 479         psDest->psDevVirtAddr.uiAddr += uiOffset;
 480         psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
 481
 482         DevmemReleaseDevVirtAddr(psSrcMemDesc);
 483 }
 484
 485
 486 void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
 487 {
 488         DevmemReleaseDevVirtAddr(psSrc);
 489 }
 490
 491 struct _RGX_SERVER_COMMON_CONTEXT_ {
 492         PVRSRV_RGXDEV_INFO *psDevInfo;
 493         DEVMEM_MEMDESC *psFWCommonContextMemDesc;
 494         PRGXFWIF_FWCOMMONCONTEXT sFWCommonContextFWAddr;
 495         DEVMEM_MEMDESC *psFWMemContextMemDesc;
 496         DEVMEM_MEMDESC *psFWFrameworkMemDesc;
 497         DEVMEM_MEMDESC *psContextStateMemDesc;
 498         RGX_CLIENT_CCB *psClientCCB;
 499         DEVMEM_MEMDESC *psClientCCBMemDesc;
 500         DEVMEM_MEMDESC *psClientCCBCtrlMemDesc;
 501         IMG_BOOL bCommonContextMemProvided;
 502         IMG_UINT32 ui32ContextID;
 503         DLLIST_NODE sListNode;
 504         RGXFWIF_CONTEXT_RESET_REASON eLastResetReason;
 505         IMG_UINT32 ui32LastResetJobRef;
 506 };
 507
 508 PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
 509                                                                          PVRSRV_DEVICE_NODE *psDeviceNode,
 510                                                                          RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
 511                                                                          RGXFWIF_DM eDM,
 512                                                                          DEVMEM_MEMDESC *psAllocatedMemDesc,
 513                                                                          IMG_UINT32 ui32AllocatedOffset,
 514                                                                          DEVMEM_MEMDESC *psFWMemContextMemDesc,
 515                                                                          DEVMEM_MEMDESC *psContextStateMemDesc,
 516                                                                          IMG_UINT32 ui32CCBAllocSize,
 517                                                                          IMG_UINT32 ui32Priority,
 518                                                                          RGX_COMMON_CONTEXT_INFO *psInfo,
 519                                                                          RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext)
 520 {
 521         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
 522         RGX_SERVER_COMMON_CONTEXT *psServerCommonContext;
 523         RGXFWIF_FWCOMMONCONTEXT *psFWCommonContext;
 524         IMG_UINT32 ui32FWCommonContextOffset;
 525         IMG_UINT8 *pui8Ptr;
 526         PVRSRV_ERROR eError;
 527
 528         /*
 529                 Allocate all the resources that are required
 530         */
 531         psServerCommonContext = OSAllocMem(sizeof(*psServerCommonContext));
 532         if (psServerCommonContext == NULL)
 533         {
 534                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 535                 goto fail_alloc;
 536         }
 537
 538         psServerCommonContext->psDevInfo = psDevInfo;
 539
 540         if (psAllocatedMemDesc)
 541         {
 542                 PDUMPCOMMENT("Using existing MemDesc for Rogue firmware %s context (offset = %d)",
 543                                          aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
 544                                          ui32AllocatedOffset);
 545                 ui32FWCommonContextOffset = ui32AllocatedOffset;
 546                 psServerCommonContext->psFWCommonContextMemDesc = psAllocatedMemDesc;
 547                 psServerCommonContext->bCommonContextMemProvided = IMG_TRUE;
 548         }
 549         else
 550         {
 551                 /* Allocate device memory for the firmware context */
 552                 PDUMPCOMMENT("Allocate Rogue firmware %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
 553                 eError = DevmemFwAllocate(psDevInfo,
 554                                                                 sizeof(*psFWCommonContext),
 555                                                                 RGX_FWCOMCTX_ALLOCFLAGS,
 556                                                                 "FwContext",
 557                                                                 &psServerCommonContext->psFWCommonContextMemDesc);
 558                 if (eError != PVRSRV_OK)
 559                 {
 560                         PVR_DPF((PVR_DBG_ERROR,"%s : Failed to allocate firmware %s context (%s)",
 561                                                                         __FUNCTION__,
 562                                                                         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
 563                                                                         PVRSRVGetErrorStringKM(eError)));
 564                         goto fail_contextalloc;
 565                 }
 566                 ui32FWCommonContextOffset = 0;
 567                 psServerCommonContext->bCommonContextMemProvided = IMG_FALSE;
 568         }
 569
 570         /* Record this context so we can refer to it if the FW needs to tell us it was reset. */
 571         psServerCommonContext->eLastResetReason    = RGXFWIF_CONTEXT_RESET_REASON_NONE;
 572         psServerCommonContext->ui32LastResetJobRef = 0;
 573         psServerCommonContext->ui32ContextID       = psDevInfo->ui32CommonCtxtCurrentID++;
 574
 575         /* Allocate the client CCB */
 576         eError = RGXCreateCCB(psDevInfo,
 577                                                   ui32CCBAllocSize,
 578                                                   psConnection,
 579                                                   eRGXCCBRequestor,
 580                                                   psServerCommonContext,
 581                                                   &psServerCommonContext->psClientCCB,
 582                                                   &psServerCommonContext->psClientCCBMemDesc,
 583                                                   &psServerCommonContext->psClientCCBCtrlMemDesc);
 584         if (eError != PVRSRV_OK)
 585         {
 586                 PVR_DPF((PVR_DBG_ERROR, "%s: failed to create CCB for %s context(%s)",
 587                                                                 __FUNCTION__,
 588                                                                 aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
 589                                                                 PVRSRVGetErrorStringKM(eError)));
 590                 goto fail_allocateccb;
 591         }
 592
 593         /*
 594                 Temporarily map the firmware context to the kernel and init it
 595         */
 596         eError = DevmemAcquireCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc,
 597                                       (void **)&pui8Ptr);
 598         if (eError != PVRSRV_OK)
 599         {
 600                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware %s context (%s)to CPU",
 601                                                                 __FUNCTION__,
 602                                                                 aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
 603                                                                 PVRSRVGetErrorStringKM(eError)));
 604                 goto fail_cpuvirtacquire;
 605         }
 606
 607         psFWCommonContext = (RGXFWIF_FWCOMMONCONTEXT *) (pui8Ptr + ui32FWCommonContextOffset);
 608         psFWCommonContext->eDM = eDM;
 609
 610         /* Set the firmware CCB device addresses in the firmware common context */
 611         RGXSetFirmwareAddress(&psFWCommonContext->psCCB,
 612                                                   psServerCommonContext->psClientCCBMemDesc,
 613                                                   0, RFW_FWADDR_FLAG_NONE);
 614         RGXSetFirmwareAddress(&psFWCommonContext->psCCBCtl,
 615                                                   psServerCommonContext->psClientCCBCtrlMemDesc,
 616                                                   0, RFW_FWADDR_FLAG_NONE);
 617
 618         if (psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_META_DMA_BIT_MASK)
 619         {
 620                 RGXSetMetaDMAAddress(&psFWCommonContext->sCCBMetaDMAAddr,
 621                                                          psServerCommonContext->psClientCCBMemDesc,
 622                                                          &psFWCommonContext->psCCB,
 623                                                          0);
 624         }
 625
 626         /* Set the memory context device address */
 627         psServerCommonContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
 628         RGXSetFirmwareAddress(&psFWCommonContext->psFWMemContext,
 629                                                   psFWMemContextMemDesc,
 630                                                   0, RFW_FWADDR_FLAG_NONE);
 631
 632         /* Set the framework register updates address */
 633         psServerCommonContext->psFWFrameworkMemDesc = psInfo->psFWFrameworkMemDesc;
 634         if (psInfo->psFWFrameworkMemDesc != NULL)
 635         {
 636                 RGXSetFirmwareAddress(&psFWCommonContext->psRFCmd,
 637                                                           psInfo->psFWFrameworkMemDesc,
 638                                                           0, RFW_FWADDR_FLAG_NONE);
 639         }
 640         else
 641         {
 642                 /* This should never be touched in this contexts without a framework
 643                  * memdesc, but ensure it is zero so we see crashes if it is.
 644                  */
 645                 psFWCommonContext->psRFCmd.ui32Addr = 0;
 646         }
 647
 648         psFWCommonContext->ui32Priority = ui32Priority;
 649         psFWCommonContext->ui32PrioritySeqNum = 0;
 650
 651         if(psInfo->psMCUFenceAddr != NULL)
 652         {
 653                 psFWCommonContext->ui64MCUFenceAddr = psInfo->psMCUFenceAddr->uiAddr;
 654         }
 655
 656         if((psDevInfo->sDevFeatureCfg.ui32CtrlStreamFormat == 2) && \
 657                                 (psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_SIGNAL_SNOOPING_BIT_MASK))
 658         {
 659                 if (eDM == RGXFWIF_DM_CDM)
 660                 {
 661                         if(psInfo->psResumeSignalAddr != NULL)
 662                         {
 663                                 psFWCommonContext->ui64ResumeSignalAddr = psInfo->psResumeSignalAddr->uiAddr;
 664                         }
 665                 }
 666         }
 667
 668         /* Store a references to Server Common Context and PID for notifications back from the FW. */
 669         psFWCommonContext->ui32ServerCommonContextID = psServerCommonContext->ui32ContextID;
 670         psFWCommonContext->ui32PID                   = OSGetCurrentClientProcessIDKM();
 671
 672         /* Set the firmware GPU context state buffer */
 673         psServerCommonContext->psContextStateMemDesc = psContextStateMemDesc;
 674         if (psContextStateMemDesc)
 675         {
 676                 RGXSetFirmwareAddress(&psFWCommonContext->psContextState,
 677                                                           psContextStateMemDesc,
 678                                                           0,
 679                                                           RFW_FWADDR_FLAG_NONE);
 680         }
 681
 682         /*
 683          * Dump the created context
 684          */
 685         PDUMPCOMMENT("Dump %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
 686         DevmemPDumpLoadMem(psServerCommonContext->psFWCommonContextMemDesc,
 687                                            ui32FWCommonContextOffset,
 688                                            sizeof(*psFWCommonContext),
 689                                            PDUMP_FLAGS_CONTINUOUS);
 690
 691         /* We've finished the setup so release the CPU mapping */
 692         DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
 693
 694         /* Map this allocation into the FW */
 695         RGXSetFirmwareAddress(&psServerCommonContext->sFWCommonContextFWAddr,
 696                                                   psServerCommonContext->psFWCommonContextMemDesc,
 697                                                   ui32FWCommonContextOffset,
 698                                                   RFW_FWADDR_FLAG_NONE);
 699
 700 #if defined(LINUX)
 701         {
 702                 IMG_UINT32 ui32FWAddr = 0;
 703                 switch (eDM) {
 704                         case RGXFWIF_DM_TA:
 705                                 ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
 706                                                 psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, sTAContext));
 707                                 break;
 708                         case RGXFWIF_DM_3D:
 709                                 ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
 710                                                 psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, s3DContext));
 711                                 break;
 712                         default:
 713                                 ui32FWAddr = psServerCommonContext->sFWCommonContextFWAddr.ui32Addr;
 714                                 break;
 715                 }
 716
 717                 trace_rogue_create_fw_context(OSGetCurrentClientProcessNameKM(),
 718                                                                           aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
 719                                                                           ui32FWAddr);
 720         }
 721 #endif
 722         /*Add the node to the list when finalised */
 723         dllist_add_to_tail(&(psDevInfo->sCommonCtxtListHead), &(psServerCommonContext->sListNode));
 724
 725         *ppsServerCommonContext = psServerCommonContext;
 726         return PVRSRV_OK;
 727
 728 fail_allocateccb:
 729         DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
 730 fail_cpuvirtacquire:
 731         RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
 732         if (!psServerCommonContext->bCommonContextMemProvided)
 733         {
 734                 DevmemFwFree(psDevInfo, psServerCommonContext->psFWCommonContextMemDesc);
 735                 psServerCommonContext->psFWCommonContextMemDesc = NULL;
 736         }
 737 fail_contextalloc:
 738         OSFreeMem(psServerCommonContext);
 739 fail_alloc:
 740         return eError;
 741 }
 742
 743 void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 744 {
 745
 746         /* Remove the context from the list of all contexts. */
 747         dllist_remove_node(&psServerCommonContext->sListNode);
 748
 749         /*
 750                 Unmap the context itself and then all it's resources
 751         */
 752
 753         /* Unmap the FW common context */
 754         RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
 755         /* Umap context state buffer (if there was one) */
 756         if (psServerCommonContext->psContextStateMemDesc)
 757         {
 758                 RGXUnsetFirmwareAddress(psServerCommonContext->psContextStateMemDesc);
 759         }
 760         /* Unmap the framework buffer */
 761         if (psServerCommonContext->psFWFrameworkMemDesc)
 762         {
 763                 RGXUnsetFirmwareAddress(psServerCommonContext->psFWFrameworkMemDesc);
 764         }
 765         /* Unmap client CCB and CCB control */
 766         RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
 767         RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
 768         /* Unmap the memory context */
 769         RGXUnsetFirmwareAddress(psServerCommonContext->psFWMemContextMemDesc);
 770
 771         /* Destroy the client CCB */
 772         RGXDestroyCCB(psServerCommonContext->psDevInfo, psServerCommonContext->psClientCCB);
 773
 774
 775         /* Free the FW common context (if there was one) */
 776         if (!psServerCommonContext->bCommonContextMemProvided)
 777         {
 778                 DevmemFwFree(psServerCommonContext->psDevInfo,
 779                                                 psServerCommonContext->psFWCommonContextMemDesc);
 780                 psServerCommonContext->psFWCommonContextMemDesc = NULL;
 781         }
 782         /* Free the hosts representation of the common context */
 783         OSFreeMem(psServerCommonContext);
 784 }
 785
 786 PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 787 {
 788         return psServerCommonContext->sFWCommonContextFWAddr;
 789 }
 790
 791 RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 792 {
 793         return psServerCommonContext->psClientCCB;
 794 }
 795
 796 RGXFWIF_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
 797                                                                IMG_UINT32 *pui32LastResetJobRef)
 798 {
 799         RGXFWIF_CONTEXT_RESET_REASON  eLastResetReason;
 800
 801         PVR_ASSERT(psServerCommonContext != NULL);
 802         PVR_ASSERT(pui32LastResetJobRef != NULL);
 803
 804         /* Take the most recent reason & job ref and reset for next time... */
 805         eLastResetReason      = psServerCommonContext->eLastResetReason;
 806         *pui32LastResetJobRef = psServerCommonContext->ui32LastResetJobRef;
 807         psServerCommonContext->eLastResetReason = RGXFWIF_CONTEXT_RESET_REASON_NONE;
 808         psServerCommonContext->ui32LastResetJobRef = 0;
 809
 810         return eLastResetReason;
 811 }
 812
 813 /*!
 814 *******************************************************************************
 815  @Function              RGXFreeKernelCCB
 816  @Description   Free the kernel CCB
 817  @Input                 psDevInfo
 818
 819  @Return                PVRSRV_ERROR
 820 ******************************************************************************/
 821 static void RGXFreeKernelCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
 822 {
 823         if (psDevInfo->psKernelCCBMemDesc != NULL)
 824         {
 825                 if (psDevInfo->psKernelCCB != NULL)
 826                 {
 827                         DevmemReleaseCpuVirtAddr(psDevInfo->psKernelCCBMemDesc);
 828                         psDevInfo->psKernelCCB = NULL;
 829                 }
 830                 DevmemFwFree(psDevInfo, psDevInfo->psKernelCCBMemDesc);
 831                 psDevInfo->psKernelCCBMemDesc = NULL;
 832         }
 833         if (psDevInfo->psKernelCCBCtlMemDesc != NULL)
 834         {
 835                 if (psDevInfo->psKernelCCBCtl != NULL)
 836                 {
 837                         DevmemReleaseCpuVirtAddr(psDevInfo->psKernelCCBCtlMemDesc);
 838                         psDevInfo->psKernelCCBCtl = NULL;
 839                 }
 840                 DevmemFwFree(psDevInfo, psDevInfo->psKernelCCBCtlMemDesc);
 841                 psDevInfo->psKernelCCBCtlMemDesc = NULL;
 842         }
 843 }
 844
 845 /*!
 846 *******************************************************************************
 847  @Function              RGXSetupKernelCCB
 848  @Description   Allocate and initialise the kernel CCB
 849  @Input                 psDevInfo
 850  @Input                 psRGXFWInit
 851  @Input                 ui32NumCmdsLog2
 852  @Input                 ui32CmdSize
 853
 854  @Return                PVRSRV_ERROR
 855 ******************************************************************************/
 856 static PVRSRV_ERROR RGXSetupKernelCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
 857                                       RGXFWIF_INIT       *psRGXFWInit,
 858                                       IMG_UINT32         ui32NumCmdsLog2,
 859                                       IMG_UINT32         ui32CmdSize)
 860 {
 861         PVRSRV_ERROR            eError;
 862         RGXFWIF_CCB_CTL         *psKCCBCtl;
 863         DEVMEM_FLAGS_T          uiCCBCtlMemAllocFlags, uiCCBMemAllocFlags;
 864         IMG_UINT32                      ui32kCCBSize = (1U << ui32NumCmdsLog2);
 865
 866
 867         /*
 868          * FIXME: the write offset need not be writeable by the firmware, indeed may
 869          * not even be needed for reading. Consider moving it to its own data
 870          * structure.
 871          */
 872         uiCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 873                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 874                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 875                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 876                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 877                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 878                                                         PVRSRV_MEMALLOCFLAG_UNCACHED |
 879                                                          PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 880
 881         /* Allocation flags for Kernel CCB */
 882         uiCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 883                                                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
 884                                                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 885                                                  PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 886                                                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 887                                                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 888                                                  PVRSRV_MEMALLOCFLAG_UNCACHED |
 889                                                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 890
 891         /*
 892          * Allocate memory for the kernel CCB control.
 893          */
 894         PDUMPCOMMENT("Allocate memory for kernel CCB control");
 895         eError = DevmemFwAllocate(psDevInfo,
 896                                   sizeof(RGXFWIF_CCB_CTL),
 897                                   uiCCBCtlMemAllocFlags,
 898                                   "FwKernelCCBControl",
 899                                   &psDevInfo->psKernelCCBCtlMemDesc);
 900
 901         if (eError != PVRSRV_OK)
 902         {
 903                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to allocate kernel CCB ctl (%u)", eError));
 904                 goto fail;
 905         }
 906
 907         /*
 908          * Allocate memory for the kernel CCB.
 909          * (this will reference further command data in non-shared CCBs)
 910          */
 911         PDUMPCOMMENT("Allocate memory for kernel CCB");
 912         eError = DevmemFwAllocate(psDevInfo,
 913                                   ui32kCCBSize * ui32CmdSize,
 914                                   uiCCBMemAllocFlags,
 915                                   "FwKernelCCB",
 916                                   &psDevInfo->psKernelCCBMemDesc);
 917
 918         if (eError != PVRSRV_OK)
 919         {
 920                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to allocate kernel CCB (%u)", eError));
 921                 goto fail;
 922         }
 923
 924         /*
 925          * Map the kernel CCB control to the kernel.
 926          */
 927         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psKernelCCBCtlMemDesc,
 928                                       (void **)&psDevInfo->psKernelCCBCtl);
 929         if (eError != PVRSRV_OK)
 930         {
 931                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to acquire cpu kernel CCB Ctl (%u)", eError));
 932                 goto fail;
 933         }
 934
 935         /*
 936          * Map the kernel CCB to the kernel.
 937          */
 938         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psKernelCCBMemDesc,
 939                                       (void **)&psDevInfo->psKernelCCB);
 940         if (eError != PVRSRV_OK)
 941         {
 942                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to acquire cpu kernel CCB (%u)", eError));
 943                 goto fail;
 944         }
 945
 946         /*
 947          * Initialise the kernel CCB control.
 948          */
 949         psKCCBCtl = psDevInfo->psKernelCCBCtl;
 950         psKCCBCtl->ui32WriteOffset = 0;
 951         psKCCBCtl->ui32ReadOffset = 0;
 952         psKCCBCtl->ui32WrapMask = ui32kCCBSize - 1;
 953         psKCCBCtl->ui32CmdSize = ui32CmdSize;
 954
 955         /*
 956          * Set-up RGXFWIfCtl pointers to access the kCCB
 957          */
 958         RGXSetFirmwareAddress(&psRGXFWInit->psKernelCCBCtl,
 959                               psDevInfo->psKernelCCBCtlMemDesc,
 960                               0, RFW_FWADDR_NOREF_FLAG);
 961
 962         RGXSetFirmwareAddress(&psRGXFWInit->psKernelCCB,
 963                               psDevInfo->psKernelCCBMemDesc,
 964                               0, RFW_FWADDR_NOREF_FLAG);
 965
 966         /*
 967          * Pdump the kernel CCB control.
 968          */
 969         PDUMPCOMMENT("Initialise kernel CCB ctl");
 970         DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc, 0, sizeof(RGXFWIF_CCB_CTL), 0);
 971
 972         return PVRSRV_OK;
 973
 974 fail:
 975         RGXFreeKernelCCB(psDevInfo);
 976
 977         PVR_ASSERT(eError != PVRSRV_OK);
 978         return eError;
 979 }
 980
 981 /*!
 982 *******************************************************************************
 983  @Function              RGXFreeFirmwareCCB
 984  @Description   Free the firmware CCB
 985  @Input                 psDevInfo
 986  @Input                 ppsFirmwareCCBCtl
 987  @Input                 ppsFirmwareCCBCtlMemDesc
 988  @Input                 ppui8FirmwareCCB
 989  @Input                 ppsFirmwareCCBMemDesc
 990
 991  @Return                void
 992 ******************************************************************************/
 993 static void RGXFreeFirmwareCCB(PVRSRV_RGXDEV_INFO       *psDevInfo,
 994                                                            RGXFWIF_CCB_CTL              **ppsFirmwareCCBCtl,
 995                                                            DEVMEM_MEMDESC               **ppsFirmwareCCBCtlMemDesc,
 996                                                            IMG_UINT8                    **ppui8FirmwareCCB,
 997                                                            DEVMEM_MEMDESC               **ppsFirmwareCCBMemDesc)
 998 {
 999         if (*ppsFirmwareCCBMemDesc != NULL)
1000         {
1001                 if (*ppui8FirmwareCCB != NULL)
1002                 {
1003                         DevmemReleaseCpuVirtAddr(*ppsFirmwareCCBMemDesc);
1004                         *ppui8FirmwareCCB = NULL;
1005                 }
1006                 DevmemFwFree(psDevInfo, *ppsFirmwareCCBMemDesc);
1007                 *ppsFirmwareCCBMemDesc = NULL;
1008         }
1009         if (*ppsFirmwareCCBCtlMemDesc != NULL)
1010         {
1011                 if (*ppsFirmwareCCBCtl != NULL)
1012                 {
1013                         DevmemReleaseCpuVirtAddr(*ppsFirmwareCCBCtlMemDesc);
1014                         *ppsFirmwareCCBCtl = NULL;
1015                 }
1016                 DevmemFwFree(psDevInfo, *ppsFirmwareCCBCtlMemDesc);
1017                 *ppsFirmwareCCBCtlMemDesc = NULL;
1018         }
1019 }
1020
1021 #define INPUT_STR_SIZE_MAX 13
1022 #define APPEND_STR_SIZE 7
1023 #define COMBINED_STR_LEN_MAX (INPUT_STR_SIZE_MAX + APPEND_STR_SIZE + 1)
1024
1025 /*!
1026 *******************************************************************************
1027  @Function              RGXSetupFirmwareCCB
1028  @Description   Allocate and initialise a Firmware CCB
1029  @Input                 psDevInfo
1030  @Input                 ppsFirmwareCCBCtl
1031  @Input                 ppsFirmwareCCBCtlMemDesc
1032  @Input                 ppui8FirmwareCCB
1033  @Input                 ppsFirmwareCCBMemDesc
1034  @Input                 psFirmwareCCBCtlFWAddr
1035  @Input                 psFirmwareCCBFWAddr
1036  @Input                 ui32NumCmdsLog2
1037  @Input                 ui32CmdSize
1038  @Input                 pszName                   Must be less than or equal to
1039                                           INPUT_STR_SIZE_MAX
1040  @Return                PVRSRV_ERROR
1041 ******************************************************************************/
1042 static PVRSRV_ERROR RGXSetupFirmwareCCB(PVRSRV_RGXDEV_INFO              *psDevInfo,
1043                                                                                 RGXFWIF_CCB_CTL                 **ppsFirmwareCCBCtl,
1044                                                                                 DEVMEM_MEMDESC                  **ppsFirmwareCCBCtlMemDesc,
1045                                                                                 IMG_UINT8                               **ppui8FirmwareCCB,
1046                                                                                 DEVMEM_MEMDESC                  **ppsFirmwareCCBMemDesc,
1047                                                                                 PRGXFWIF_CCB_CTL                *psFirmwareCCBCtlFWAddr,
1048                                                                                 PRGXFWIF_CCB                    *psFirmwareCCBFWAddr,
1049                                                                                 IMG_UINT32                              ui32NumCmdsLog2,
1050                                                                                 IMG_UINT32                              ui32CmdSize,
1051                                                                                 IMG_PCHAR                               pszName)
1052 {
1053         PVRSRV_ERROR            eError;
1054         RGXFWIF_CCB_CTL         *psFWCCBCtl;
1055         DEVMEM_FLAGS_T          uiCCBCtlMemAllocFlags, uiCCBMemAllocFlags;
1056         IMG_UINT32                      ui32FWCCBSize = (1U << ui32NumCmdsLog2);
1057         IMG_CHAR                        sCCBCtlName[COMBINED_STR_LEN_MAX] = "";
1058         IMG_CHAR                        sAppend[] = "Control";
1059
1060         /*
1061          * FIXME: the write offset need not be writeable by the host, indeed may
1062          * not even be needed for reading. Consider moving it to its own data
1063          * structure.
1064          */
1065         uiCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1066                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1067                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1068                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1069                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1070                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1071                                                         PVRSRV_MEMALLOCFLAG_UNCACHED |
1072                                                          PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1073
1074         /* Allocation flags for Firmware CCB */
1075         uiCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1076                                                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1077                                                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1078                                                  PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1079                                                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1080                                                  PVRSRV_MEMALLOCFLAG_UNCACHED |
1081                                                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1082
1083         PVR_ASSERT(strlen(sCCBCtlName) == 0);
1084         PVR_ASSERT(strlen(sAppend) == APPEND_STR_SIZE);
1085         PVR_ASSERT(strlen(pszName) <= INPUT_STR_SIZE_MAX);
1086
1087         /* Append "Control" to the name for the control struct. */
1088         strncat(sCCBCtlName, pszName, INPUT_STR_SIZE_MAX);
1089         strncat(sCCBCtlName, sAppend, APPEND_STR_SIZE);
1090
1091         /*
1092                 Allocate memory for the Firmware CCB control.
1093         */
1094         PDUMPCOMMENT("Allocate memory for %s", sCCBCtlName);
1095         eError = DevmemFwAllocate(psDevInfo,
1096                                                         sizeof(RGXFWIF_CCB_CTL),
1097                                                         uiCCBCtlMemAllocFlags,
1098                                                         sCCBCtlName,
1099                             ppsFirmwareCCBCtlMemDesc);
1100
1101         if (eError != PVRSRV_OK)
1102         {
1103                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to allocate %s (%u)", sCCBCtlName,  eError));
1104                 goto fail;
1105         }
1106
1107         /*
1108                 Allocate memory for the Firmware CCB.
1109                 (this will reference further command data in non-shared CCBs)
1110         */
1111         PDUMPCOMMENT("Allocate memory for %s", pszName);
1112         eError = DevmemFwAllocate(psDevInfo,
1113                                                         ui32FWCCBSize * ui32CmdSize,
1114                                                         uiCCBMemAllocFlags,
1115                                                         pszName,
1116                             ppsFirmwareCCBMemDesc);
1117
1118         if (eError != PVRSRV_OK)
1119         {
1120                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to allocate %s (%u)", pszName, eError));
1121                 goto fail;
1122         }
1123
1124         /*
1125                 Map the Firmware CCB control to the kernel.
1126         */
1127         eError = DevmemAcquireCpuVirtAddr(*ppsFirmwareCCBCtlMemDesc,
1128                                       (void **)ppsFirmwareCCBCtl);
1129         if (eError != PVRSRV_OK)
1130         {
1131                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to acquire cpu %s (%u)", sCCBCtlName, eError));
1132                 goto fail;
1133         }
1134
1135         /*
1136                 Map the firmware CCB to the kernel.
1137         */
1138         eError = DevmemAcquireCpuVirtAddr(*ppsFirmwareCCBMemDesc,
1139                                       (void **)ppui8FirmwareCCB);
1140         if (eError != PVRSRV_OK)
1141         {
1142                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to acquire cpu %s (%u)", pszName, eError));
1143                 goto fail;
1144         }
1145
1146         /*
1147          * Initialise the firmware CCB control.
1148          */
1149         psFWCCBCtl = *ppsFirmwareCCBCtl;
1150         psFWCCBCtl->ui32WriteOffset = 0;
1151         psFWCCBCtl->ui32ReadOffset = 0;
1152         psFWCCBCtl->ui32WrapMask = ui32FWCCBSize - 1;
1153         psFWCCBCtl->ui32CmdSize = ui32CmdSize;
1154
1155         /*
1156          * Set-up RGXFWIfCtl pointers to access the kCCBs
1157          */
1158         RGXSetFirmwareAddress(psFirmwareCCBCtlFWAddr,
1159                                                   *ppsFirmwareCCBCtlMemDesc,
1160                                                   0, RFW_FWADDR_NOREF_FLAG);
1161
1162         RGXSetFirmwareAddress(psFirmwareCCBFWAddr,
1163                                                   *ppsFirmwareCCBMemDesc,
1164                                                   0, RFW_FWADDR_NOREF_FLAG);
1165
1166         /*
1167          * Pdump the kernel CCB control.
1168          */
1169         PDUMPCOMMENT("Initialise %s", sCCBCtlName);
1170         DevmemPDumpLoadMem(*ppsFirmwareCCBCtlMemDesc,
1171                                            0,
1172                                            sizeof(RGXFWIF_CCB_CTL),
1173                                            0);
1174
1175         return PVRSRV_OK;
1176
1177 fail:
1178         RGXFreeFirmwareCCB(psDevInfo,
1179                                            ppsFirmwareCCBCtl,
1180                                            ppsFirmwareCCBCtlMemDesc,
1181                                            ppui8FirmwareCCB,
1182                                            ppsFirmwareCCBMemDesc);
1183
1184         PVR_ASSERT(eError != PVRSRV_OK);
1185         return eError;
1186 }
1187
1188 static void RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
1189 {
1190         PMR *psPMR;
1191
1192         if (psDevInfo->psRGXFaultAddressMemDesc)
1193         {
1194                 if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc,(void **)&psPMR) == PVRSRV_OK)
1195                 {
1196                         PMRUnlockSysPhysAddresses(psPMR);
1197                 }
1198                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
1199                 psDevInfo->psRGXFaultAddressMemDesc = NULL;
1200         }
1201 }
1202
1203 static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE        *psDeviceNode, RGXFWIF_INIT *psRGXFWInit)
1204 {
1205         PVRSRV_ERROR            eError = PVRSRV_OK;
1206 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
1207         IMG_UINT32                      *pui32MemoryVirtAddr;
1208         IMG_UINT32                      i;
1209         size_t                  ui32PageSize;
1210         DEVMEM_FLAGS_T          uiMemAllocFlags;
1211         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1212         PMR                                     *psPMR;
1213
1214         ui32PageSize = OSGetPageSize();
1215
1216         /* Allocate page of memory to use for page faults on non-blocking memory transactions */
1217         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1218                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1219                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1220                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1221                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1222                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1223                                                 PVRSRV_MEMALLOCFLAG_UNCACHED;
1224
1225         psDevInfo->psRGXFaultAddressMemDesc = NULL;
1226         eError = DevmemFwAllocateExportable(psDeviceNode,
1227                                                                                 ui32PageSize,
1228                                                                                 ui32PageSize,
1229                                                                                 uiMemAllocFlags,
1230                                                                                 "FwExFaultAddress",
1231                                                                                 &psDevInfo->psRGXFaultAddressMemDesc);
1232
1233         if (eError != PVRSRV_OK)
1234         {
1235                 PVR_DPF((PVR_DBG_ERROR,"Failed to allocate mem for fault address (%u)",
1236                                 eError));
1237                 goto failFaultAddressDescAlloc;
1238         }
1239
1240
1241         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
1242                                                                           (void **)&pui32MemoryVirtAddr);
1243         if (eError != PVRSRV_OK)
1244         {
1245                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire mem for fault address (%u)",
1246                                 eError));
1247                 goto failFaultAddressDescAqCpuVirt;
1248         }
1249
1250         for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
1251         {
1252                 *(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
1253         }
1254
1255         eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc,(void **)&psPMR);
1256
1257         if (eError != PVRSRV_OK)
1258         {
1259                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Error getting PMR for fault address (%u)",
1260                                 eError));
1261
1262                 goto failFaultAddressDescGetPMR;
1263         }
1264         else
1265         {
1266                 IMG_BOOL bValid;
1267                 IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
1268
1269                 eError = PMRLockSysPhysAddresses(psPMR);
1270
1271                 if (eError != PVRSRV_OK)
1272                 {
1273                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Error locking physical address for fault address MemDesc (%u)",
1274                                         eError));
1275
1276                         goto failFaultAddressDescLockPhys;
1277                 }
1278
1279                 eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize,1,0,&(psRGXFWInit->sFaultPhysAddr),&bValid);
1280
1281                 if (eError != PVRSRV_OK)
1282                 {
1283                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Error getting physical address for fault address MemDesc (%u)",
1284                                         eError));
1285
1286                         goto failFaultAddressDescGetPhys;
1287                 }
1288
1289                 if (!bValid)
1290                 {
1291                         psRGXFWInit->sFaultPhysAddr.uiAddr = 0;
1292                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed getting physical address for fault address MemDesc - invalid page (0x%llX)",
1293                                         psRGXFWInit->sFaultPhysAddr.uiAddr));
1294
1295                         goto failFaultAddressDescGetPhys;
1296                 }
1297         }
1298
1299         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
1300
1301         return PVRSRV_OK;
1302
1303 failFaultAddressDescGetPhys:
1304         PMRUnlockSysPhysAddresses(psPMR);
1305
1306 failFaultAddressDescLockPhys:
1307
1308 failFaultAddressDescGetPMR:
1309         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
1310
1311 failFaultAddressDescAqCpuVirt:
1312         DevmemFwFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
1313         psDevInfo->psRGXFaultAddressMemDesc = NULL;
1314
1315 failFaultAddressDescAlloc:
1316 #endif
1317         return eError;
1318 }
1319
1320 static PVRSRV_ERROR RGXHwBrn37200(PVRSRV_RGXDEV_INFO *psDevInfo)
1321 {
1322         PVRSRV_ERROR                    eError = PVRSRV_OK;
1323
1324 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
1325         IMG_UINT64      ui64ErnsBrns = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
1326         IMG_UINT32      ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(psDevInfo->sDevFeatureCfg.ui32CacheLineSize);
1327
1328         if(ui64ErnsBrns & FIX_HW_BRN_37200_BIT_MASK)
1329         {
1330                 struct _DEVMEM_HEAP_    *psBRNHeap;
1331                 DEVMEM_FLAGS_T                  uiFlags;
1332                 IMG_DEV_VIRTADDR                sTmpDevVAddr;
1333                 size_t                          uiPageSize;
1334
1335                 uiPageSize = OSGetPageSize();
1336
1337                 uiFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1338                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1339                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1340                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1341                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1342                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1343                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1344                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1345                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1346
1347                 eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx,
1348                                                                   "HWBRN37200", /* FIXME: We need to create an IDENT macro for this string.
1349                                                                                                  Make sure the IDENT macro is not accessible to userland */
1350                                                                   &psBRNHeap);
1351
1352                 if (eError != PVRSRV_OK)
1353                 {
1354                         PVR_DPF((PVR_DBG_ERROR,"RGXHwBrn37200: HWBRN37200 Failed DevmemFindHeapByName (%u)", eError));
1355                         goto failFWHWBRN37200FindHeapByName;
1356                 }
1357
1358                 psDevInfo->psRGXFWHWBRN37200MemDesc = NULL;
1359                 eError = DevmemAllocate(psBRNHeap,
1360                                                         uiPageSize,
1361                                                         ui32CacheLineSize,
1362                                                         uiFlags,
1363                                                         "HWBRN37200",
1364                                                         &psDevInfo->psRGXFWHWBRN37200MemDesc);
1365
1366                 if (eError != PVRSRV_OK)
1367                 {
1368                         PVR_DPF((PVR_DBG_ERROR,"RGXHwBrn37200: Failed to allocate %u bytes for HWBRN37200 (%u)",
1369                                         (IMG_UINT32)uiPageSize,
1370                                         eError));
1371                         goto failFWHWBRN37200MemDescAlloc;
1372                 }
1373
1374                 /*
1375                         We need to map it so the heap for this allocation
1376                         is set
1377                 */
1378                 eError = DevmemMapToDevice(psDevInfo->psRGXFWHWBRN37200MemDesc,
1379                                                            psBRNHeap,
1380                                                            &sTmpDevVAddr);
1381
1382                 if (eError != PVRSRV_OK)
1383                 {
1384                         PVR_DPF((PVR_DBG_ERROR,"RGXHwBrn37200: Failed to allocate %u bytes for HWBRN37200 (%u)",
1385                                         (IMG_UINT32)uiPageSize,
1386                                         eError));
1387                         goto failFWHWBRN37200DevmemMapToDevice;
1388                 }
1389
1390
1391
1392                 return PVRSRV_OK;
1393
1394         failFWHWBRN37200DevmemMapToDevice:
1395
1396         failFWHWBRN37200MemDescAlloc:
1397                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWHWBRN37200MemDesc);
1398                 psDevInfo->psRGXFWHWBRN37200MemDesc = NULL;
1399
1400         failFWHWBRN37200FindHeapByName:;
1401         }
1402 #endif
1403         return eError;
1404 }
1405
1406 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
1407 /*************************************************************************/ /*!
1408 @Function       RGXTraceBufferIsInitRequired
1409
1410 @Description    Returns true if the firmware trace buffer is not allocated and
1411                 might be required by the firmware soon. Trace buffer allocated
1412                 on-demand to reduce RAM footprint on systems not needing
1413                 firmware trace.
1414
1415 @Input          psDevInfo        RGX device info
1416
1417 @Return         IMG_BOOL        Whether on-demand allocation(s) is/are needed
1418                                 or not
1419 */ /**************************************************************************/
1420 INLINE IMG_BOOL RGXTraceBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
1421 {
1422         RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
1423
1424         /* The firmware expects a trace buffer only when:
1425          *      - Logtype is "trace" AND
1426          *      - at least one LogGroup is configured
1427          */
1428         if((psDevInfo->psRGXFWIfTraceBufferMemDesc[0] == NULL)
1429                 && (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)
1430                 && (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK))
1431         {
1432                 return IMG_TRUE;
1433         }
1434
1435         return IMG_FALSE;
1436 }
1437
1438 /*************************************************************************/ /*!
1439 @Function       RGXTraceBufferInitOnDemandResources
1440
1441 @Description    Allocates the firmware trace buffer required for dumping trace
1442                 info from the firmware.
1443
1444 @Input          psDevInfo        RGX device info
1445
1446 @Return         PVRSRV_OK       If all went good, PVRSRV_ERROR otherwise.
1447 */ /**************************************************************************/
1448 PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo)
1449 {
1450         RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
1451         DEVMEM_FLAGS_T     uiMemAllocFlags;
1452         PVRSRV_ERROR       eError = PVRSRV_OK;
1453         IMG_UINT32         ui32FwThreadNum;
1454
1455         uiMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1456                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1457                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1458                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1459                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1460                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1461                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1462
1463         for (ui32FwThreadNum = 0; ui32FwThreadNum < RGXFW_THREAD_NUM; ui32FwThreadNum++)
1464         {
1465                 /* Ensure allocation API is only called when not already allocated */
1466                 PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum] == NULL);
1467
1468                 PDUMPCOMMENT("Allocate rgxfw trace buffer(%u)", ui32FwThreadNum);
1469                 eError = DevmemFwAllocate(psDevInfo,
1470                                                 RGXFW_TRACE_BUFFER_SIZE * sizeof(*(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32TraceBuffer)),
1471                                                 uiMemAllocFlags,
1472                                                 "FwTraceBuffer",
1473                                                 &psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum]);
1474                 if (eError != PVRSRV_OK)
1475                 {
1476                         PVR_DPF((PVR_DBG_ERROR,"%s: Failed to allocate %zu bytes for fw trace buffer %u (Error code:%u)",
1477                                         __FUNCTION__,
1478                                         RGXFW_TRACE_BUFFER_SIZE * sizeof(*(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32TraceBuffer)),
1479                                         ui32FwThreadNum,
1480                                         eError));
1481                         goto fail;
1482                 }
1483
1484                 /* Firmware address should not be already set */
1485                 PVR_ASSERT(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer.ui32Addr == 0x0);
1486
1487                 /* for the FW to use this address when dumping in log (trace) buffer */
1488                 RGXSetFirmwareAddress(&psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer,
1489                                                 psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
1490                                                 0, RFW_FWADDR_NOREF_FLAG);
1491                 /* Set an address for the host to be able to read fw trace buffer */
1492                 eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
1493                                                                 (void **)&psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32TraceBuffer);
1494                 if (eError != PVRSRV_OK)
1495                 {
1496                         PVR_DPF((PVR_DBG_ERROR,"%s: Failed to acquire kernel tracebuf (%u) ctl (Error code: %u)",
1497                                         __FUNCTION__, ui32FwThreadNum, eError));
1498                         goto fail;
1499                 }
1500         }
1501
1502 /* Just return error in-case of failures, clean-up would be handled by DeInit function */
1503 fail:
1504         return eError;
1505 }
1506
1507 /*************************************************************************/ /*!
1508 @Function       RGXTraceBufferDeinit
1509
1510 @Description    Deinitialises all the allocations and references that are made
1511                 for the FW trace buffer(s)
1512
1513 @Input          ppsDevInfo       RGX device info
1514 @Return         void
1515 */ /**************************************************************************/
1516 static void RGXTraceBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
1517 {
1518         RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
1519         IMG_UINT32 i;
1520
1521         for (i = 0; i < RGXFW_THREAD_NUM; i++)
1522         {
1523                 if (psDevInfo->psRGXFWIfTraceBufferMemDesc[i])
1524                 {
1525                         if (psTraceBufCtl->sTraceBuf[i].pui32TraceBuffer != NULL)
1526                         {
1527                                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
1528                                 psTraceBufCtl->sTraceBuf[i].pui32TraceBuffer = NULL;
1529                         }
1530
1531                         DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
1532                         psDevInfo->psRGXFWIfTraceBufferMemDesc[i] = NULL;
1533                 }
1534         }
1535 }
1536 #endif
1537
1538 /*!
1539 *******************************************************************************
1540
1541  @Function      RGXSetupFirmware
1542
1543  @Description
1544
1545  Setups all the firmware related data
1546
1547  @Input psDevInfo
1548
1549  @Return PVRSRV_ERROR
1550
1551 ******************************************************************************/
1552 PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
1553                               IMG_BOOL                 bEnableSignatureChecks,
1554                               IMG_UINT32               ui32SignatureChecksBufSize,
1555                               IMG_UINT32               ui32HWPerfFWBufSizeKB,
1556                               IMG_UINT64               ui64HWPerfFilter,
1557                               IMG_UINT32               ui32RGXFWAlignChecksArrLength,
1558                               IMG_UINT32               *pui32RGXFWAlignChecks,
1559                               IMG_UINT32               ui32ConfigFlags,
1560                               IMG_UINT32               ui32LogType,
1561                               RGXFWIF_BIFTILINGMODE    eBifTilingMode,
1562                               IMG_UINT32               ui32NumTilingCfgs,
1563                               IMG_UINT32               *pui32BIFTilingXStrides,
1564                               IMG_UINT32               ui32FilterFlags,
1565                               IMG_UINT32               ui32JonesDisableMask,
1566                               IMG_UINT32               ui32HWRDebugDumpLimit,
1567                               IMG_UINT32               ui32HWPerfCountersDataSize,
1568                               PMR                      **ppsHWPerfPMR,
1569                               RGXFWIF_DEV_VIRTADDR     *psRGXFWInitFWAddr,
1570                               RGX_RD_POWER_ISLAND_CONF eRGXRDPowerIslandConf,
1571                               FW_PERF_CONF             eFirmwarePerf)
1572
1573 {
1574         PVRSRV_ERROR            eError;
1575         DEVMEM_FLAGS_T          uiMemAllocFlags;
1576         RGXFWIF_INIT            *psRGXFWInit = NULL;
1577         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1578         IMG_UINT32                      dm, ui32Temp = 0;
1579         IMG_UINT64                      ui64ErnsBrns;
1580 #if defined (SUPPORT_PDVFS)
1581         RGXFWIF_PDVFS_OPP   *psPDVFSOPPInfo;
1582         IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg;
1583 #endif
1584         ui64ErnsBrns = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
1585
1586         /* Fw init data */
1587
1588         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1589                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
1590                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1591                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1592                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1593                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1594                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1595                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1596                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1597                                                 /* FIXME: Change to Cached */
1598
1599
1600         PDUMPCOMMENT("Allocate RGXFWIF_INIT structure");
1601
1602         eError = DevmemFwAllocate(psDevInfo,
1603                                                         sizeof(RGXFWIF_INIT),
1604                                                         uiMemAllocFlags,
1605                                                         "FwInitStructure",
1606                                                         &psDevInfo->psRGXFWIfInitMemDesc);
1607
1608         if (eError != PVRSRV_OK)
1609         {
1610                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for fw if ctl (%u)",
1611                                 (IMG_UINT32)sizeof(RGXFWIF_INIT),
1612                                 eError));
1613                 goto fail;
1614         }
1615
1616         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc,
1617                                                                           (void **)&psRGXFWInit);
1618         if (eError != PVRSRV_OK)
1619         {
1620                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel fw if ctl (%u)",
1621                                 eError));
1622                 goto fail;
1623         }
1624
1625         RGXSetFirmwareAddress(&psDevInfo->sFWInitFWAddr,
1626                               psDevInfo->psRGXFWIfInitMemDesc,
1627                               0, RFW_FWADDR_NOREF_FLAG);
1628         *psRGXFWInitFWAddr = psDevInfo->sFWInitFWAddr;
1629
1630 #if defined(PVRSRV_GPUVIRT_GUESTDRV)
1631         /*
1632          * Guest drivers do not support the following functionality:
1633          *  - Perform actual on-chip fw loading & initialisation
1634          *  - Perform actual on-chip fw management (i.e. reset)
1635          *      - Perform actual on-chip fw HWPerf,Trace,Utils,ActivePM
1636          */
1637 #else
1638         /* FW trace control structure */
1639         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1640                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1641                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1642                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1643                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1644                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1645                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1646
1647         PDUMPCOMMENT("Allocate rgxfw trace control structure");
1648         eError = DevmemFwAllocate(psDevInfo,
1649                                                         sizeof(RGXFWIF_TRACEBUF),
1650                                                         uiMemAllocFlags,
1651                                                         "FwTraceCtlStruct",
1652                                                         &psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
1653
1654         if (eError != PVRSRV_OK)
1655         {
1656                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for fw trace (%u)",
1657                                 (IMG_UINT32)sizeof(RGXFWIF_TRACEBUF),
1658                                 eError));
1659                 goto fail;
1660         }
1661
1662         RGXSetFirmwareAddress(&psRGXFWInit->sTraceBufCtl,
1663                                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
1664                                                 0, RFW_FWADDR_NOREF_FLAG);
1665
1666         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
1667                                                                           (void **)&psDevInfo->psRGXFWIfTraceBuf);
1668         if (eError != PVRSRV_OK)
1669         {
1670                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel tracebuf ctl (%u)",
1671                                 eError));
1672                 goto fail;
1673         }
1674
1675         /* Set initial firmware log type/group(s) */
1676         if (ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
1677         {
1678                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1679                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Invalid initial log type (0x%X)",ui32LogType));
1680                 goto fail;
1681         }
1682         psDevInfo->psRGXFWIfTraceBuf->ui32LogType = ui32LogType;
1683
1684 #if defined (PDUMP)
1685         /* When PDUMP is enabled, ALWAYS allocate on-demand trace buffer resource
1686          * (irrespective of loggroup(s) enabled), given that logtype/loggroups can
1687          * be set during PDump playback in logconfig, at any point of time */
1688         eError = RGXTraceBufferInitOnDemandResources(psDevInfo);
1689 #else
1690         /* Otherwise, allocate only if required */
1691         if (RGXTraceBufferIsInitRequired(psDevInfo))
1692         {
1693                 eError = RGXTraceBufferInitOnDemandResources(psDevInfo);
1694         }
1695         else
1696         {
1697                 eError = PVRSRV_OK;
1698         }
1699 #endif
1700         PVR_LOGG_IF_ERROR(eError, "RGXTraceBufferInitOnDemandResources", fail);
1701
1702         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1703                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1704                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1705                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1706                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1707                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1708
1709         if ((0 != psDevInfo->sDevFeatureCfg.ui32MCMS) && \
1710                         (0 == (ui64ErnsBrns & FIX_HW_BRN_50767_BIT_MASK)))
1711         {
1712                 IMG_BOOL bMetaDMA = psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_META_DMA_BIT_MASK;
1713
1714 #if defined(SUPPORT_TRUSTED_DEVICE)
1715                 if (bMetaDMA)
1716                 {
1717                         IMG_UINT64 ui64SecBufHandle;
1718
1719                         PDUMPCOMMENT("Import secure buffer to store FW coremem data");
1720                         eError = DevmemImportTDSecureBuf(psDeviceNode,
1721                                                          RGX_META_COREMEM_BSS_SIZE,
1722                                                          OSGetPageShift(),
1723                                                          uiMemAllocFlags,
1724                                                          &psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
1725                                                          &ui64SecBufHandle);
1726                 }
1727                 else
1728 #endif
1729                 {
1730                         PDUMPCOMMENT("Allocate buffer to store FW coremem data");
1731                         eError = DevmemFwAllocate(psDevInfo,
1732                                                                           RGX_META_COREMEM_BSS_SIZE,
1733                                                                           uiMemAllocFlags,
1734                                                                           "FwCorememDataStore",
1735                                                                           &psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
1736                 }
1737
1738                 if (eError != PVRSRV_OK)
1739                 {
1740                         PVR_DPF((PVR_DBG_ERROR, "RGXSetupFirmware: Failed to allocate coremem data store (%u)",
1741                                          eError));
1742                         goto fail;
1743                 }
1744
1745                 RGXSetFirmwareAddress(&psRGXFWInit->sCorememDataStore.pbyFWAddr,
1746                                                           psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
1747                                                           0, RFW_FWADDR_NOREF_FLAG);
1748
1749                 if (bMetaDMA)
1750                 {
1751                         RGXSetMetaDMAAddress(&psRGXFWInit->sCorememDataStore,
1752                                                                  psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
1753                                                                  &psRGXFWInit->sCorememDataStore.pbyFWAddr,
1754                                                                  0);
1755                 }
1756         }
1757
1758         /* init HW frame info */
1759         PDUMPCOMMENT("Allocate rgxfw HW info buffer");
1760         eError = DevmemFwAllocate(psDevInfo,
1761                                                         sizeof(RGXFWIF_HWRINFOBUF),
1762                                                         uiMemAllocFlags,
1763                                                         "FwHWInfoBuffer",
1764                                                         &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
1765
1766         if (eError != PVRSRV_OK)
1767         {
1768                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %d bytes for HW info (%u)",
1769                                 (IMG_UINT32)sizeof(RGXFWIF_HWRINFOBUF),
1770                                 eError));
1771                 goto fail;
1772         }
1773
1774         RGXSetFirmwareAddress(&psRGXFWInit->sRGXFWIfHWRInfoBufCtl,
1775                                                 psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
1776                                                 0, RFW_FWADDR_NOREF_FLAG);
1777
1778         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
1779                                                                           (void **)&psDevInfo->psRGXFWIfHWRInfoBuf);
1780         if (eError != PVRSRV_OK)
1781         {
1782                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel tracebuf ctl (%u)",
1783                                 eError));
1784                 goto fail;
1785         }
1786
1787         /* Might be uncached. Be conservative and use a DeviceMemSet */
1788         OSDeviceMemSet(psDevInfo->psRGXFWIfHWRInfoBuf, 0, sizeof(RGXFWIF_HWRINFOBUF));
1789
1790         /* Allocate shared buffer for GPU utilisation */
1791         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1792                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1793                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1794                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1795                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1796                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1797                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1798
1799         PDUMPCOMMENT("Allocate shared buffer for GPU utilisation");
1800         eError = DevmemFwAllocate(psDevInfo,
1801                                                         sizeof(RGXFWIF_GPU_UTIL_FWCB),
1802                                                         uiMemAllocFlags,
1803                                                         "FwGPUUtilisationBuffer",
1804                                                         &psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc);
1805
1806         if (eError != PVRSRV_OK)
1807         {
1808                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for GPU utilisation buffer ctl (%u)",
1809                                 (IMG_UINT32)sizeof(RGXFWIF_GPU_UTIL_FWCB),
1810                                 eError));
1811                 goto fail;
1812         }
1813
1814         RGXSetFirmwareAddress(&psRGXFWInit->sGpuUtilFWCbCtl,
1815                                                 psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc,
1816                                                 0, RFW_FWADDR_NOREF_FLAG);
1817
1818         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc,
1819                                                                           (void **)&psDevInfo->psRGXFWIfGpuUtilFWCb);
1820         if (eError != PVRSRV_OK)
1821         {
1822                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel GPU utilisation buffer ctl (%u)",
1823                                 eError));
1824                 goto fail;
1825         }
1826
1827         /* Initialise GPU utilisation buffer */
1828         psDevInfo->psRGXFWIfGpuUtilFWCb->ui64LastWord =
1829             RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(),RGXFWIF_GPU_UTIL_STATE_IDLE);
1830
1831
1832         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1833                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1834                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1835                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1836                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1837                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1838                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1839
1840         PDUMPCOMMENT("Allocate rgxfw FW runtime configuration (FW)");
1841         eError = DevmemFwAllocate(psDevInfo,
1842                                                         sizeof(RGXFWIF_RUNTIME_CFG),
1843                                                         uiMemAllocFlags,
1844                                                         "FwRuntimeCfg",
1845                                                         &psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
1846
1847         if (eError != PVRSRV_OK)
1848         {
1849                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for FW runtime configuration (%u)",
1850                                 (IMG_UINT32)sizeof(RGXFWIF_RUNTIME_CFG),
1851                                 eError));
1852                 goto fail;
1853         }
1854
1855         RGXSetFirmwareAddress(&psRGXFWInit->sRuntimeCfg,
1856                                                 psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
1857                                                 0, RFW_FWADDR_NOREF_FLAG);
1858
1859         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
1860                                                                         (void **)&psDevInfo->psRGXFWIfRuntimeCfg);
1861         if (eError != PVRSRV_OK)
1862         {
1863                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel FW runtime configuration (%u)",
1864                                 eError));
1865                 goto fail;
1866         }
1867
1868
1869         /* HWPerf: Determine the size of the FW buffer */
1870         if (ui32HWPerfFWBufSizeKB == 0 ||
1871                         ui32HWPerfFWBufSizeKB == RGXFW_HWPERF_L1_SIZE_DEFAULT)
1872         {
1873                 /* Under pvrsrvctl 0 size implies AppHint not set or is set to zero,
1874                  * use default size from driver constant.  Under SUPPORT_KERNEL_SRVINIT
1875                  * default is the above macro. In either case, set it to the default,
1876                  * size, no logging.
1877                  */
1878                 psDevInfo->ui32RGXFWIfHWPerfBufSize = RGXFW_HWPERF_L1_SIZE_DEFAULT<<10;
1879         }
1880         else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX))
1881         {
1882                 /* Size specified as a AppHint but it is too big */
1883                 PVR_DPF((PVR_DBG_WARNING,"RGXSetupFirmware: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
1884                                 ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX));
1885                 psDevInfo->ui32RGXFWIfHWPerfBufSize = RGXFW_HWPERF_L1_SIZE_MAX<<10;
1886         }
1887         else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN))
1888         {
1889                 /* Size specified as in AppHint HWPerfFWBufSizeInKB */
1890                 PVR_DPF((PVR_DBG_WARNING,"RGXSetupFirmware: Using HWPerf FW buffer size of %u KB",
1891                                 ui32HWPerfFWBufSizeKB));
1892                 psDevInfo->ui32RGXFWIfHWPerfBufSize = ui32HWPerfFWBufSizeKB<<10;
1893         }
1894         else
1895         {
1896                 /* Size specified as a AppHint but it is too small */
1897                 PVR_DPF((PVR_DBG_WARNING,"RGXSetupFirmware: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
1898                                 ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN));
1899                 psDevInfo->ui32RGXFWIfHWPerfBufSize = RGXFW_HWPERF_L1_SIZE_MIN<<10;
1900         }
1901
1902         /* init HWPERF data */
1903         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfRIdx = 0;
1904         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfWIdx = 0;
1905         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfWrapCount = 0;
1906         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
1907         psRGXFWInit->bDisableFilterHWPerfCustomCounter = (ui32ConfigFlags & RGXFWIF_INICFG_HWP_DISABLE_FILTER) ? IMG_TRUE : IMG_FALSE;
1908         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfUt = 0;
1909         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfDropCount = 0;
1910         psDevInfo->psRGXFWIfTraceBuf->ui32FirstDropOrdinal = 0;
1911         psDevInfo->psRGXFWIfTraceBuf->ui32LastDropOrdinal = 0;
1912         psDevInfo->psRGXFWIfTraceBuf->ui32PowMonEnergy = 0;
1913
1914         /* Second stage initialisation or HWPerf, hHWPerfLock created in first
1915          * stage. See RGXRegisterDevice() call to RGXHWPerfInit(). */
1916         if (psDevInfo->ui64HWPerfFilter == 0)
1917         {
1918                 psDevInfo->ui64HWPerfFilter = ui64HWPerfFilter;
1919                 psRGXFWInit->ui64HWPerfFilter = ui64HWPerfFilter;
1920         }
1921         else
1922         {
1923                 /* The filter has already been modified. This can happen if the driver
1924                  * was compiled with SUPPORT_KERNEL_SRVINIT enabled and e.g.
1925                  * pvr/gpu_tracing_on was enabled. */
1926                 psRGXFWInit->ui64HWPerfFilter = psDevInfo->ui64HWPerfFilter;
1927         }
1928
1929 #if defined (PDUMP)
1930         /* When PDUMP is enabled, ALWAYS allocate on-demand HWPerf resources
1931          * (irrespective of HWPerf enabled or not), given that HWPerf can be
1932          * enabled during PDump playback via RTCONF at any point of time. */
1933         eError = RGXHWPerfInitOnDemandResources();
1934 #else
1935         /* Otherwise, only allocate if HWPerf is enabled via apphint */
1936         if (ui32ConfigFlags & RGXFWIF_INICFG_HWPERF_EN)
1937         {
1938                 eError = RGXHWPerfInitOnDemandResources();
1939         }
1940 #endif
1941         PVR_LOGG_IF_ERROR(eError, "RGXHWPerfInitOnDemandResources", fail);
1942
1943         RGXHWPerfInitAppHintCallbacks(psDeviceNode);
1944
1945 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
1946         PDUMPCOMMENT("Allocate rgxfw register configuration structure");
1947         eError = DevmemFwAllocate(psDevInfo,
1948                                                         sizeof(RGXFWIF_REG_CFG),
1949                                                         uiMemAllocFlags | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
1950                                                         "FwRegisterConfigStructure",
1951                                                         &psDevInfo->psRGXFWIfRegCfgMemDesc);
1952
1953         if (eError != PVRSRV_OK)
1954         {
1955                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for fw register configurations (%u)",
1956                                 (IMG_UINT32)sizeof(RGXFWIF_REG_CFG),
1957                                 eError));
1958                 goto fail;
1959         }
1960
1961         RGXSetFirmwareAddress(&psRGXFWInit->sRegCfg,
1962                                                 psDevInfo->psRGXFWIfRegCfgMemDesc,
1963                                                 0, RFW_FWADDR_NOREF_FLAG);
1964 #endif
1965
1966         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1967                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1968                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1969                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1970                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1971                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1972                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1973
1974         PDUMPCOMMENT("Allocate rgxfw hwperfctl structure");
1975         eError = DevmemFwAllocateExportable(psDeviceNode,
1976                                                         ui32HWPerfCountersDataSize,
1977                                                         OSGetPageSize(),
1978                                                         uiMemAllocFlags,
1979                                                         "FwExHWPerfControlStructure",
1980                                                         &psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
1981
1982         if (eError != PVRSRV_OK)
1983         {
1984                 PVR_DPF((PVR_DBG_ERROR,"RGXInitHWPerfCounters: Failed to allocate %u bytes for fw hwperf control (%u)",
1985                                 ui32HWPerfCountersDataSize,
1986                                 eError));
1987                 goto fail;
1988         }
1989
1990         eError = DevmemLocalGetImportHandle(psDevInfo->psRGXFWIfHWPerfCountersMemDesc, (void**) ppsHWPerfPMR);
1991
1992         if (eError != PVRSRV_OK)
1993         {
1994                 PVR_DPF((PVR_DBG_ERROR,"DevmemLocalGetImportHandle failed (%u)", eError));
1995                 goto fail;
1996         }
1997
1998
1999         RGXSetFirmwareAddress(&psRGXFWInit->sHWPerfCtl,
2000                                                 psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
2001                                                 0, 0);
2002
2003         /* Required info by FW to calculate the ActivePM idle timer latency */
2004         {
2005                 RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
2006                 RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
2007
2008                 psRGXFWInit->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
2009                 psRGXFWInit->ui32ActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
2010
2011                 /* Initialise variable runtime configuration to the system defaults */
2012                 psRuntimeCfg->ui32CoreClockSpeed = psRGXFWInit->ui32InitialCoreClockSpeed;
2013                 psRuntimeCfg->ui32ActivePMLatencyms = psRGXFWInit->ui32ActivePMLatencyms;
2014                 psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
2015
2016                 /* Initialize the DefaultDustsNumInit Field to Max Dusts */
2017                 psRuntimeCfg->ui32DefaultDustsNumInit = MAX(1, (psDevInfo->sDevFeatureCfg.ui32NumClusters/2));
2018         }
2019 #if defined(PDUMP)
2020         PDUMPCOMMENT("Dump initial state of FW runtime configuration");
2021         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
2022                                                 0,
2023                                                 sizeof(RGXFWIF_RUNTIME_CFG),
2024                                                 PDUMP_FLAGS_CONTINUOUS);
2025 #endif
2026 #endif /* defined(PVRSRV_GPUVIRT_GUESTDRV) */
2027
2028         /* Allocate a sync for power management */
2029         eError = SyncPrimContextCreate(psDevInfo->psDeviceNode,
2030                                        &psDevInfo->hSyncPrimContext);
2031         if (eError != PVRSRV_OK)
2032         {
2033                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate sync primitive context with error (%u)", eError));
2034                 goto fail;
2035         }
2036
2037         eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
2038         if (eError != PVRSRV_OK)
2039         {
2040                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate sync primitive with error (%u)", eError));
2041                 goto fail;
2042         }
2043
2044         eError = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim,
2045                         &psRGXFWInit->sPowerSync.ui32Addr);
2046         if (eError != PVRSRV_OK)
2047         {
2048                 PVR_DPF((PVR_DBG_ERROR,
2049                         "%s: Failed to get Sync Prim FW address with error (%u)",
2050                         __FUNCTION__, eError));
2051                 goto fail;
2052         }
2053
2054         /* Setup Fault read register */
2055         eError = RGXSetupFaultReadRegister(psDeviceNode, psRGXFWInit);
2056         if (eError != PVRSRV_OK)
2057         {
2058                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup fault read register"));
2059                 goto fail;
2060         }
2061
2062         /* Apply FIX_HW_BRN_37200 */
2063         eError = RGXHwBrn37200(psDevInfo);
2064         if (eError != PVRSRV_OK)
2065         {
2066                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to apply HWBRN37200"));
2067                 goto fail;
2068         }
2069
2070 #if defined(SUPPORT_PVRSRV_GPUVIRT)
2071         if(!(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK))
2072         {
2073                 ui32Temp = RGXFWIF_KCCB_NUMCMDS_LOG2_GPUVIRT_ONLY;
2074         }else
2075 #endif
2076         {
2077                 ui32Temp = RGXFWIF_KCCB_NUMCMDS_LOG2_FEAT_GPU_VIRTUALISATION;
2078         }
2079         /*
2080          * Set up kernel CCB.
2081          */
2082         eError = RGXSetupKernelCCB(psDevInfo,
2083                                    psRGXFWInit,
2084                                    ui32Temp,
2085                                    sizeof(RGXFWIF_KCCB_CMD));
2086         if (eError != PVRSRV_OK)
2087         {
2088                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Kernel CCB"));
2089                 goto fail;
2090         }
2091
2092         /*
2093          * Set up firmware CCB.
2094          */
2095         eError = RGXSetupFirmwareCCB(psDevInfo,
2096                                                                  &psDevInfo->psFirmwareCCBCtl,
2097                                                                  &psDevInfo->psFirmwareCCBCtlMemDesc,
2098                                                                  &psDevInfo->psFirmwareCCB,
2099                                                                  &psDevInfo->psFirmwareCCBMemDesc,
2100                                                                  &psRGXFWInit->psFirmwareCCBCtl,
2101                                                                  &psRGXFWInit->psFirmwareCCB,
2102                                                                  RGXFWIF_FWCCB_NUMCMDS_LOG2,
2103                                                                  sizeof(RGXFWIF_FWCCB_CMD),
2104                                                                  "FwCCB");
2105
2106         if (eError != PVRSRV_OK)
2107         {
2108                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware CCB"));
2109                 goto fail;
2110         }
2111         /* RD Power Island */
2112         {
2113                 RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
2114                 IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
2115                 IMG_BOOL bEnableRDPowIsland = ((eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
2116                                                 (eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
2117
2118                 ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
2119         }
2120
2121 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2122         ui32ConfigFlags |= RGXFWIF_INICFG_WORKEST_V2;
2123
2124 #if defined(SUPPORT_PDVFS)
2125         /* Proactive DVFS depends on Workload Estimation */
2126         psPDVFSOPPInfo = &(psRGXFWInit->sPDVFSOPPInfo);
2127         psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
2128
2129         if(psDVFSDeviceCfg->pasOPPTable != NULL)
2130         {
2131                 if(psDVFSDeviceCfg->ui32OPPTableSize >
2132                    sizeof(psPDVFSOPPInfo->asOPPValues)/sizeof(psPDVFSOPPInfo->asOPPValues[0]))
2133                 {
2134                         PVR_DPF((PVR_DBG_ERROR,
2135                                  "RGXSetupFirmware: OPP Table too large :"
2136                                  " Size = %u, Maximum size = %lu",
2137                                  psDVFSDeviceCfg->ui32OPPTableSize,
2138                                  (unsigned long)(sizeof(psPDVFSOPPInfo->asOPPValues)/sizeof(psPDVFSOPPInfo->asOPPValues[0]))));
2139                         eError = PVRSRV_ERROR_INVALID_PARAMS;
2140                         goto fail;
2141                 }
2142
2143                 memcpy(psPDVFSOPPInfo->asOPPValues,
2144                            psDVFSDeviceCfg->pasOPPTable,
2145                            sizeof(psPDVFSOPPInfo->asOPPValues));
2146                 psPDVFSOPPInfo->ui32MaxOPPPoint =
2147                         (psDVFSDeviceCfg->ui32OPPTableSize) - 1;
2148
2149                 ui32ConfigFlags |= RGXFWIF_INICFG_PDVFS_V2;
2150         }
2151         else
2152         {
2153                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Missing OPP Table"));
2154         }
2155 #endif
2156 #endif
2157
2158         psRGXFWInit->ui32ConfigFlags = ui32ConfigFlags & RGXFWIF_INICFG_ALL;
2159
2160 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2161         /*
2162          * Set up Workload Estimation firmware CCB.
2163          */
2164         eError = RGXSetupFirmwareCCB(psDevInfo,
2165                                                                  &psDevInfo->psWorkEstFirmwareCCBCtl,
2166                                                                  &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
2167                                                                  &psDevInfo->psWorkEstFirmwareCCB,
2168                                                                  &psDevInfo->psWorkEstFirmwareCCBMemDesc,
2169                                                                  &psRGXFWInit->psWorkEstFirmwareCCBCtl,
2170                                                                  &psRGXFWInit->psWorkEstFirmwareCCB,
2171                                                                  RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2,
2172                                                                  sizeof(RGXFWIF_WORKEST_FWCCB_CMD),
2173                                                                  "FwWEstCCB");
2174
2175         if (eError != PVRSRV_OK)
2176         {
2177                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Workload Estimation Firmware CCB"));
2178                 goto fail;
2179         }
2180 #endif
2181
2182         /* Require a minimum amount of memory for the signature buffers */
2183         if (ui32SignatureChecksBufSize < RGXFW_SIG_BUFFER_SIZE_MIN)
2184         {
2185                 ui32SignatureChecksBufSize = RGXFW_SIG_BUFFER_SIZE_MIN;
2186         }
2187
2188         /* Setup Signature and Checksum Buffers for TA and 3D */
2189         eError = RGXFWSetupSignatureChecks(psDevInfo,
2190                                            &psDevInfo->psRGXFWSigTAChecksMemDesc,
2191                                            ui32SignatureChecksBufSize,
2192                                            &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_TA],
2193                                            "TA");
2194         if (eError != PVRSRV_OK)
2195         {
2196                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup TA signature checks"));
2197                 goto fail;
2198         }
2199         psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
2200
2201         eError = RGXFWSetupSignatureChecks(psDevInfo,
2202                                            &psDevInfo->psRGXFWSig3DChecksMemDesc,
2203                                            ui32SignatureChecksBufSize,
2204                                            &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_3D],
2205                                            "3D");
2206         if (eError != PVRSRV_OK)
2207         {
2208                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup 3D signature checks"));
2209                 goto fail;
2210         }
2211         psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
2212
2213         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_RAY_TRACING_BIT_MASK)
2214         {
2215                 eError = RGXFWSetupSignatureChecks(psDevInfo,
2216                                                                                    &psDevInfo->psRGXFWSigRTChecksMemDesc,
2217                                                                                    ui32SignatureChecksBufSize,
2218                                                                                    &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_RTU],
2219                                                                                    "RTU");
2220                 if (eError != PVRSRV_OK)
2221                 {
2222                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup RTU signature checks"));
2223                         goto fail;
2224                 }
2225                 psDevInfo->ui32SigRTChecksSize = ui32SignatureChecksBufSize;
2226
2227                 eError = RGXFWSetupSignatureChecks(psDevInfo,
2228                                                                                    &psDevInfo->psRGXFWSigSHChecksMemDesc,
2229                                                                                    ui32SignatureChecksBufSize,
2230                                                                                    &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_SHG],
2231                                                                                    "SHG");
2232                 if (eError != PVRSRV_OK)
2233                 {
2234                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup SHG signature checks"));
2235                         goto fail;
2236                 }
2237                 psDevInfo->ui32SigSHChecksSize = ui32SignatureChecksBufSize;
2238         }
2239
2240 #if defined(RGXFW_ALIGNCHECKS)
2241         eError = RGXFWSetupAlignChecks(psDevInfo,
2242                                                                 &psRGXFWInit->sAlignChecks,
2243                                                                 pui32RGXFWAlignChecks,
2244                                                                 ui32RGXFWAlignChecksArrLength);
2245         if (eError != PVRSRV_OK)
2246         {
2247                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup alignment checks"));
2248                 goto fail;
2249         }
2250 #endif
2251
2252         psRGXFWInit->ui32FilterFlags = ui32FilterFlags;
2253
2254 #if defined(PVRSRV_GPUVIRT_GUESTDRV)
2255         /*
2256          * Guest drivers do not support the following functionality:
2257          *  - Perform actual on-chip fw RDPowIsland(ing)
2258          *  - Perform actual on-chip fw tracing
2259          *  - Configure FW perf counters
2260          */
2261         PVR_UNREFERENCED_PARAMETER(dm);
2262         PVR_UNREFERENCED_PARAMETER(eFirmwarePerf);
2263 #else
2264
2265         if(ui64ErnsBrns & FIX_HW_BRN_52402_BIT_MASK)
2266         {
2267                 /* Fill the remaining bits of fw the init data */
2268                 psRGXFWInit->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_BRN_52402_HEAP_BASE;
2269                 psRGXFWInit->sUSCExecBase.uiAddr = RGX_USCCODE_BRN_52402_HEAP_BASE;
2270         }else
2271         {
2272                 /* Fill the remaining bits of fw the init data */
2273                 psRGXFWInit->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
2274                 psRGXFWInit->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
2275         }
2276
2277         psRGXFWInit->sDPXControlStreamBase.uiAddr = RGX_DOPPLER_HEAP_BASE;
2278         psRGXFWInit->sResultDumpBase.uiAddr = RGX_DOPPLER_OVERFLOW_HEAP_BASE;
2279         psRGXFWInit->sRTUHeapBase.uiAddr = RGX_DOPPLER_HEAP_BASE;
2280         psRGXFWInit->sTDMTPUYUVCeoffsHeapBase.uiAddr = RGX_TDM_TPU_YUV_COEFFS_HEAP_BASE;
2281
2282
2283         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_S7_TOP_INFRASTRUCTURE_BIT_MASK)
2284         {
2285                 psRGXFWInit->ui32JonesDisableMask = ui32JonesDisableMask;
2286         }
2287         psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_SRVCFG_DISABLE_PDP_EN)
2288                         ? IMG_FALSE : IMG_TRUE;
2289         psRGXFWInit->ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
2290
2291         psRGXFWInit->eFirmwarePerf = eFirmwarePerf;
2292
2293         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_SLC_VIVT_BIT_MASK)
2294         {
2295                 eError = _AllocateSLC3Fence(psDevInfo, psRGXFWInit);
2296                 if (eError != PVRSRV_OK)
2297                 {
2298                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate memory for SLC3Fence"));
2299                         goto fail;
2300                 }
2301         }
2302
2303
2304         if ( (psDevInfo->sDevFeatureCfg.ui32META) && \
2305                         ((ui32ConfigFlags & RGXFWIF_INICFG_METAT1_ENABLED) != 0))
2306         {
2307                 /* Allocate a page for T1 stack */
2308                 eError = DevmemFwAllocate(psDevInfo,
2309                                           RGX_META_STACK_SIZE,
2310                                           RGX_FWCOMCTX_ALLOCFLAGS,
2311                                           "FwMETAT1Stack",
2312                                           & psDevInfo->psMETAT1StackMemDesc);
2313                 if (eError != PVRSRV_OK)
2314                 {
2315                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate T1 Stack"));
2316                         goto fail;
2317                 }
2318
2319                 RGXSetFirmwareAddress(&psRGXFWInit->sT1Stack,
2320                                       psDevInfo->psMETAT1StackMemDesc,
2321                                       0, RFW_FWADDR_NOREF_FLAG);
2322
2323                 PVR_DPF((PVR_DBG_MESSAGE, "RGXSetupFirmware: T1 Stack Frame allocated at %x",
2324                                  psRGXFWInit->sT1Stack.ui32Addr));
2325         }
2326
2327 #if defined(SUPPORT_PDVFS)
2328                 /* Core clock rate */
2329                 uiMemAllocFlags =
2330                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
2331                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
2332                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
2333                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
2334                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
2335                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
2336                         PVRSRV_MEMALLOCFLAG_UNCACHED |
2337                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
2338
2339                 eError = DevmemFwAllocate(psDevInfo,
2340                                                                   sizeof(IMG_UINT32),
2341                                                                   uiMemAllocFlags,
2342                                                                   "FwCoreClkRate",
2343                                                                   &psDevInfo->psRGXFWIFCoreClkRateMemDesc);
2344                 if (eError != PVRSRV_OK)
2345                 {
2346                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate PDVFS core clock rate"));
2347                         goto fail;
2348                 }
2349
2350                 RGXSetFirmwareAddress(&psRGXFWInit->sCoreClockRate,
2351                                                           psDevInfo->psRGXFWIFCoreClkRateMemDesc,
2352                                                           0, RFW_FWADDR_NOREF_FLAG);
2353
2354                 PVR_DPF((PVR_DBG_MESSAGE, "RGXSetupFirmware: PDVFS core clock rate allocated at %x",
2355                                  psRGXFWInit->sCoreClockRate.ui32Addr));
2356
2357                 eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc,
2358                                                                                   (void **)&psDevInfo->pui32RGXFWIFCoreClkRate);
2359                 if (eError != PVRSRV_OK)
2360                 {
2361                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire core clk rate (%u)",
2362                                         eError));
2363                         goto fail;
2364                 }
2365 #endif
2366
2367         /* Timestamps */
2368         uiMemAllocFlags =
2369                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
2370                 PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
2371                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
2372                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
2373                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
2374                 PVRSRV_MEMALLOCFLAG_UNCACHED |
2375                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
2376
2377         /*
2378           the timer query arrays
2379         */
2380         PDUMPCOMMENT("Allocate timer query arrays (FW)");
2381         eError = DevmemFwAllocate(psDevInfo,
2382                                   sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
2383                                   uiMemAllocFlags,
2384                                   "FwStartTimesArray",
2385                                   & psDevInfo->psStartTimeMemDesc);
2386         if (eError != PVRSRV_OK)
2387         {
2388                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map start times array"));
2389                 goto fail;
2390         }
2391
2392         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
2393                                           (void **)& psDevInfo->pui64StartTimeById);
2394
2395         if (eError != PVRSRV_OK)
2396         {
2397                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map start times array"));
2398                 goto fail;
2399         }
2400
2401         eError = DevmemFwAllocate(psDevInfo,
2402                                   sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
2403                                   uiMemAllocFlags,
2404                                   "FwEndTimesArray",
2405                                   & psDevInfo->psEndTimeMemDesc);
2406         if (eError != PVRSRV_OK)
2407         {
2408                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map end times array"));
2409                 goto fail;
2410         }
2411
2412         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
2413                                           (void **)& psDevInfo->pui64EndTimeById);
2414
2415         if (eError != PVRSRV_OK)
2416         {
2417                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map end times array"));
2418                 goto fail;
2419         }
2420
2421         eError = DevmemFwAllocate(psDevInfo,
2422                                   sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
2423                                   uiMemAllocFlags,
2424                                   "FwCompletedOpsArray",
2425                                   & psDevInfo->psCompletedMemDesc);
2426         if (eError != PVRSRV_OK)
2427         {
2428                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to completed ops array"));
2429                 goto fail;
2430         }
2431
2432         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
2433                                           (void **)& psDevInfo->pui32CompletedById);
2434
2435         if (eError != PVRSRV_OK)
2436         {
2437                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map completed ops array"));
2438                 goto fail;
2439         }
2440
2441                 /* Initialize FW started flag */
2442         psRGXFWInit->bFirmwareStarted = IMG_FALSE;
2443         psRGXFWInit->ui32MarkerVal = 1;
2444
2445         /* Initialise the compatibility check data */
2446         RGXFWIF_COMPCHECKS_BVNC_INIT(psRGXFWInit->sRGXCompChecks.sFWBVNC);
2447         RGXFWIF_COMPCHECKS_BVNC_INIT(psRGXFWInit->sRGXCompChecks.sHWBVNC);
2448
2449         PDUMPCOMMENT("Dump RGXFW Init data");
2450         if (!bEnableSignatureChecks)
2451         {
2452 #if defined(PDUMP)
2453                 PDUMPCOMMENT("(to enable rgxfw signatures place the following line after the RTCONF line)");
2454                 DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfInitMemDesc,
2455                                                         offsetof(RGXFWIF_INIT, asSigBufCtl),
2456                                                         sizeof(RGXFWIF_SIGBUF_CTL)*(psDevInfo->sDevFeatureCfg.ui32MAXDMCount),
2457                                                         PDUMP_FLAGS_CONTINUOUS);
2458 #endif
2459                 psRGXFWInit->asSigBufCtl[RGXFWIF_DM_3D].sBuffer.ui32Addr = 0x0;
2460                 psRGXFWInit->asSigBufCtl[RGXFWIF_DM_TA].sBuffer.ui32Addr = 0x0;
2461         }
2462
2463         for (dm = 0; dm < (psDevInfo->sDevFeatureCfg.ui32MAXDMCount); dm++)
2464         {
2465                 psDevInfo->psRGXFWIfTraceBuf->aui32HwrDmLockedUpCount[dm] = 0;
2466                 psDevInfo->psRGXFWIfTraceBuf->aui32HwrDmOverranCount[dm] = 0;
2467                 psDevInfo->psRGXFWIfTraceBuf->aui32HwrDmRecoveredCount[dm] = 0;
2468                 psDevInfo->psRGXFWIfTraceBuf->aui32HwrDmFalseDetectCount[dm] = 0;
2469         }
2470
2471         /*
2472          * BIF Tiling configuration
2473          */
2474
2475         psRGXFWInit->eBifTilingMode = eBifTilingMode;
2476
2477         psRGXFWInit->sBifTilingCfg[0].uiBase = RGX_BIF_TILING_HEAP_1_BASE;
2478         psRGXFWInit->sBifTilingCfg[0].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2479         psRGXFWInit->sBifTilingCfg[0].uiXStride = pui32BIFTilingXStrides[0];
2480         psRGXFWInit->sBifTilingCfg[1].uiBase = RGX_BIF_TILING_HEAP_2_BASE;
2481         psRGXFWInit->sBifTilingCfg[1].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2482         psRGXFWInit->sBifTilingCfg[1].uiXStride = pui32BIFTilingXStrides[1];
2483         psRGXFWInit->sBifTilingCfg[2].uiBase = RGX_BIF_TILING_HEAP_3_BASE;
2484         psRGXFWInit->sBifTilingCfg[2].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2485         psRGXFWInit->sBifTilingCfg[2].uiXStride = pui32BIFTilingXStrides[2];
2486         psRGXFWInit->sBifTilingCfg[3].uiBase = RGX_BIF_TILING_HEAP_4_BASE;
2487         psRGXFWInit->sBifTilingCfg[3].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2488         psRGXFWInit->sBifTilingCfg[3].uiXStride = pui32BIFTilingXStrides[3];
2489
2490 #if defined(PDUMP)
2491         PDUMPCOMMENT("Dump rgxfw hwperfctl structure");
2492         DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
2493                                 0,
2494                                                         ui32HWPerfCountersDataSize,
2495                                 PDUMP_FLAGS_CONTINUOUS);
2496
2497         PDUMPCOMMENT("Dump rgxfw trace control structure");
2498         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
2499                                                 0,
2500                                                 sizeof(RGXFWIF_TRACEBUF),
2501                                                 PDUMP_FLAGS_CONTINUOUS);
2502 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
2503         PDUMPCOMMENT("Dump rgxfw register configuration buffer");
2504         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfRegCfgMemDesc,
2505                                                 0,
2506                                                 sizeof(RGXFWIF_REG_CFG),
2507                                                 PDUMP_FLAGS_CONTINUOUS);
2508 #endif
2509         PDUMPCOMMENT("Dump rgxfw init structure");
2510         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfInitMemDesc,
2511                                                 0,
2512                                                 sizeof(RGXFWIF_INIT),
2513                                                 PDUMP_FLAGS_CONTINUOUS);
2514         if ((0 != psDevInfo->sDevFeatureCfg.ui32MCMS) && \
2515                                 (0 == (psDevInfo->sDevFeatureCfg.ui64ErnsBrns & FIX_HW_BRN_50767_BIT_MASK)))
2516         {
2517                 PDUMPCOMMENT("Dump rgxfw coremem data store");
2518                 DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
2519                                                         0,
2520                                                         RGX_META_COREMEM_BSS_SIZE,
2521                                                         PDUMP_FLAGS_CONTINUOUS);
2522         }
2523
2524         PDUMPCOMMENT("RTCONF: run-time configuration");
2525
2526
2527         /* Dump the config options so they can be edited.
2528          *
2529          * FIXME: Need new DevmemPDumpWRW API which writes a WRW to load ui32ConfigFlags
2530          */
2531         PDUMPCOMMENT("(Set the FW config options here)");
2532         PDUMPCOMMENT("( Ctx Switch TA Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_TA_EN);
2533         PDUMPCOMMENT("( Ctx Switch 3D Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_3D_EN);
2534         PDUMPCOMMENT("( Ctx Switch CDM Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_CDM_EN);
2535         PDUMPCOMMENT("( Ctx Switch Rand mode: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
2536         PDUMPCOMMENT("( Ctx Switch Soft Reset Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
2537         PDUMPCOMMENT("( Reserved (do not set): 0x%08x)", RGXFWIF_INICFG_RSVD);
2538         PDUMPCOMMENT("( Rascal+Dust Power Island: 0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
2539         PDUMPCOMMENT("( Enable HWPerf: 0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
2540         PDUMPCOMMENT("( Enable HWR: 0x%08x)", RGXFWIF_INICFG_HWR_EN);
2541         PDUMPCOMMENT("( Check MList: 0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
2542         PDUMPCOMMENT("( Disable Auto Clock Gating: 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
2543         PDUMPCOMMENT("( Enable HWPerf Polling Perf Counter: 0x%08x)", RGXFWIF_INICFG_POLL_COUNTERS_EN);
2544
2545         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_VDM_OBJECT_LEVEL_LLS_BIT_MASK)
2546         {
2547                 PDUMPCOMMENT("( Ctx Switch Object mode Index: 0x%08x)", RGXFWIF_INICFG_VDM_CTX_STORE_MODE_INDEX);
2548                 PDUMPCOMMENT("( Ctx Switch Object mode Instance: 0x%08x)", RGXFWIF_INICFG_VDM_CTX_STORE_MODE_INSTANCE);
2549                 PDUMPCOMMENT("( Ctx Switch Object mode List: 0x%08x)", RGXFWIF_INICFG_VDM_CTX_STORE_MODE_LIST);
2550         }
2551
2552         PDUMPCOMMENT("( Enable SHG Bypass mode: 0x%08x)", RGXFWIF_INICFG_SHG_BYPASS_EN);
2553         PDUMPCOMMENT("( Enable RTU Bypass mode: 0x%08x)", RGXFWIF_INICFG_RTU_BYPASS_EN);
2554         PDUMPCOMMENT("( Enable register configuration: 0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
2555         PDUMPCOMMENT("( Assert on TA Out-of-Memory: 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
2556         PDUMPCOMMENT("( Disable HWPerf custom counter filter: 0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
2557         PDUMPCOMMENT("( Enable HWPerf custom performance timer: 0x%08x)", RGXFWIF_INICFG_CUSTOM_PERF_TIMER_EN);
2558         PDUMPCOMMENT("( Enable CDM Killing Rand mode: 0x%08x)", RGXFWIF_INICFG_CDM_KILL_MODE_RAND_EN);
2559         PDUMPCOMMENT("( Enable Ctx Switch profile mode: 0x%08x (none=b'000, fast=b'001, medium=b'010, slow=b'011, nodelay=b'100))", RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK);
2560         PDUMPCOMMENT("( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
2561         PDUMPCOMMENT("( Enable Meta T1 running main code: 0x%08x)", RGXFWIF_INICFG_METAT1_MAIN);
2562         PDUMPCOMMENT("( Enable Meta T1 running dummy code: 0x%08x)", RGXFWIF_INICFG_METAT1_DUMMY);
2563         PDUMPCOMMENT("( Assert on HWR trigger (page fault, lockup, overrun or poll failure): 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER);
2564
2565         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2566                                                         offsetof(RGXFWIF_INIT, ui32ConfigFlags),
2567                                                         psRGXFWInit->ui32ConfigFlags,
2568                                                         PDUMP_FLAGS_CONTINUOUS);
2569
2570         /* default: no filter */
2571         psRGXFWInit->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
2572         psRGXFWInit->sPIDFilter.asItems[0].uiPID = 0;
2573
2574         PDUMPCOMMENT("( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
2575                                                         RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
2576                                                         RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
2577
2578         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2579                                                         offsetof(RGXFWIF_INIT, sPIDFilter.eMode),
2580                                                         psRGXFWInit->sPIDFilter.eMode,
2581                                                         PDUMP_FLAGS_CONTINUOUS);
2582
2583         PDUMPCOMMENT("( PID filter PID/OSID list (Up to %u entries. Terminate with a zero PID))",
2584                                                                         RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
2585         {
2586                 IMG_UINT32 i;
2587
2588                 /* generate a few WRWs in the pdump stream as an example */
2589                 for(i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
2590                 {
2591                         /*
2592                          * Some compilers cannot cope with the uses of offsetof() below - the specific problem being the use of
2593                          * a non-const variable in the expression, which it needs to be const. Typical compiler output is
2594                          * "expression must have a constant value".
2595                          */
2596                         const IMG_DEVMEM_OFFSET_T uiPIDOff
2597                         = (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_INIT *)0)->sPIDFilter.asItems[i].uiPID);
2598
2599                         const IMG_DEVMEM_OFFSET_T uiOSIDOff
2600                         = (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_INIT *)0)->sPIDFilter.asItems[i].ui32OSID);
2601
2602                         PDUMPCOMMENT("(PID and OSID pair %u)", i);
2603
2604                         PDUMPCOMMENT("(PID)");
2605                         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2606                                                 uiPIDOff,
2607                                                 0,
2608                                                 PDUMP_FLAGS_CONTINUOUS);
2609
2610                         PDUMPCOMMENT("(OSID)");
2611                         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2612                                                 uiOSIDOff,
2613                                                 0,
2614                                                 PDUMP_FLAGS_CONTINUOUS);
2615                 }
2616         }
2617
2618         /*
2619          * Dump the log config so it can be edited.
2620          */
2621         PDUMPCOMMENT("(Set the log config here)");
2622         PDUMPCOMMENT("( Log Type: set bit 0 for TRACE, reset for TBI)");
2623         PDUMPCOMMENT("( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
2624         PDUMPCOMMENT("( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
2625         PDUMPCOMMENT("( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
2626         PDUMPCOMMENT("( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
2627         PDUMPCOMMENT("( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
2628         PDUMPCOMMENT("( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
2629         PDUMPCOMMENT("( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
2630         PDUMPCOMMENT("( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
2631         PDUMPCOMMENT("( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
2632         PDUMPCOMMENT("( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
2633         PDUMPCOMMENT("( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
2634
2635         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_RAY_TRACING_BIT_MASK)
2636         {
2637                 PDUMPCOMMENT("( RPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RPM);
2638         }
2639
2640         if (psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_META_DMA_BIT_MASK)
2641         {
2642                 PDUMPCOMMENT("( DMA Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DMA);
2643         }
2644         PDUMPCOMMENT("( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
2645         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
2646                                                         offsetof(RGXFWIF_TRACEBUF, ui32LogType),
2647                                                         psDevInfo->psRGXFWIfTraceBuf->ui32LogType,
2648                                                         PDUMP_FLAGS_CONTINUOUS);
2649
2650         PDUMPCOMMENT("Set the HWPerf Filter config here");
2651         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfInitMemDesc,
2652                                                 offsetof(RGXFWIF_INIT, ui64HWPerfFilter),
2653                                                 psRGXFWInit->ui64HWPerfFilter,
2654                                                 PDUMP_FLAGS_CONTINUOUS);
2655
2656 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
2657         PDUMPCOMMENT("(Number of registers configurations at pow on, dust change, ta, 3d, cdm and tla/tdm)");
2658         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2659                                                         offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_PWR_ON]),
2660                                                         0,
2661                                                         PDUMP_FLAGS_CONTINUOUS);
2662         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2663                                                         offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_DUST_CHANGE]),
2664                                                         0,
2665                                                         PDUMP_FLAGS_CONTINUOUS);
2666         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2667                                                         offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_TA]),
2668                                                         0,
2669                                                         PDUMP_FLAGS_CONTINUOUS);
2670         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2671                                                         offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_3D]),
2672                                                         0,
2673                                                         PDUMP_FLAGS_CONTINUOUS);
2674         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2675                                                         offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_CDM]),
2676                                                         0,
2677                                                         PDUMP_FLAGS_CONTINUOUS);
2678
2679         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_TLA_BIT_MASK)
2680         {
2681                 DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2682                                                                 offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_TLA]),
2683                                                                 0,
2684                                                                 PDUMP_FLAGS_CONTINUOUS);
2685         }
2686
2687         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_FASTRENDER_DM_BIT_MASK)
2688         {
2689                 DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2690                                                         offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[RGXFWIF_REG_CFG_TYPE_TDM]),
2691                                                         0,
2692                                                         PDUMP_FLAGS_CONTINUOUS);
2693         }
2694
2695         PDUMPCOMMENT("(Set registers here: address, mask, value)");
2696         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
2697                                                         offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
2698                                                         0,
2699                                                         PDUMP_FLAGS_CONTINUOUS);
2700         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
2701                                                         offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Mask),
2702                                                         0,
2703                                                         PDUMP_FLAGS_CONTINUOUS);
2704         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
2705                                                         offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
2706                                                         0,
2707                                                         PDUMP_FLAGS_CONTINUOUS);
2708 #endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
2709 #endif /* PDUMP */
2710 #endif /* PVRSRV_GPUVIRT_GUESTDRV */
2711
2712 #if defined(SUPPORT_PVRSRV_GPUVIRT)
2713         /* Perform additional virtualisation initialisation */
2714         eError = RGXVzSetupFirmware(psDeviceNode);
2715         if (eError != PVRSRV_OK)
2716         {
2717                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed RGXVzSetupFirmware"));
2718                 goto fail;
2719         }
2720 #endif
2721
2722         /* We don't need access to the fw init data structure anymore */
2723         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
2724         psRGXFWInit = NULL;
2725
2726         psDevInfo->bFirmwareInitialised = IMG_TRUE;
2727
2728         return PVRSRV_OK;
2729
2730 fail:
2731 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
2732         if (psDevInfo->psRGXFWIfInitMemDesc != NULL && psRGXFWInit != NULL)
2733         {
2734                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
2735         }
2736 #endif
2737         RGXFreeFirmware(psDevInfo);
2738
2739         PVR_ASSERT(eError != PVRSRV_OK);
2740         return eError;
2741 }
2742
2743 /*!
2744 *******************************************************************************
2745
2746  @Function      RGXFreeFirmware
2747
2748  @Description
2749
2750  Frees all the firmware-related allocations
2751
2752  @Input psDevInfo
2753
2754  @Return PVRSRV_ERROR
2755
2756 ******************************************************************************/
2757 void RGXFreeFirmware(PVRSRV_RGXDEV_INFO         *psDevInfo)
2758 {
2759         IMG_UINT64      ui64ErnsBrns = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
2760
2761         psDevInfo->bFirmwareInitialised = IMG_FALSE;
2762
2763         RGXFreeKernelCCB(psDevInfo);
2764
2765         RGXFreeFirmwareCCB(psDevInfo,
2766                                            &psDevInfo->psFirmwareCCBCtl,
2767                                            &psDevInfo->psFirmwareCCBCtlMemDesc,
2768                                            &psDevInfo->psFirmwareCCB,
2769                                            &psDevInfo->psFirmwareCCBMemDesc);
2770
2771 #if defined(SUPPORT_WORKLOAD_ESTIMATION)
2772         RGXFreeFirmwareCCB(psDevInfo,
2773                                            &psDevInfo->psWorkEstFirmwareCCBCtl,
2774                                            &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
2775                                            &psDevInfo->psWorkEstFirmwareCCB,
2776                                            &psDevInfo->psWorkEstFirmwareCCBMemDesc);
2777 #endif
2778
2779 #if defined(RGXFW_ALIGNCHECKS)
2780         if (psDevInfo->psRGXFWAlignChecksMemDesc)
2781         {
2782                 RGXFWFreeAlignChecks(psDevInfo);
2783         }
2784 #endif
2785
2786         if (psDevInfo->psRGXFWSigTAChecksMemDesc)
2787         {
2788                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWSigTAChecksMemDesc);
2789                 psDevInfo->psRGXFWSigTAChecksMemDesc = NULL;
2790         }
2791
2792         if (psDevInfo->psRGXFWSig3DChecksMemDesc)
2793         {
2794                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWSig3DChecksMemDesc);
2795                 psDevInfo->psRGXFWSig3DChecksMemDesc = NULL;
2796         }
2797
2798         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_RAY_TRACING_BIT_MASK)
2799         {
2800                 if (psDevInfo->psRGXFWSigRTChecksMemDesc)
2801                 {
2802                         DevmemFwFree(psDevInfo, psDevInfo->psRGXFWSigRTChecksMemDesc);
2803                         psDevInfo->psRGXFWSigRTChecksMemDesc = NULL;
2804                 }
2805
2806                 if (psDevInfo->psRGXFWSigSHChecksMemDesc)
2807                 {
2808                         DevmemFwFree(psDevInfo, psDevInfo->psRGXFWSigSHChecksMemDesc);
2809                         psDevInfo->psRGXFWSigSHChecksMemDesc = NULL;
2810                 }
2811         }
2812
2813         if(ui64ErnsBrns & FIX_HW_BRN_37200_BIT_MASK)
2814         {
2815                 if (psDevInfo->psRGXFWHWBRN37200MemDesc)
2816                 {
2817                         DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWHWBRN37200MemDesc);
2818                         DevmemFree(psDevInfo->psRGXFWHWBRN37200MemDesc);
2819                         psDevInfo->psRGXFWHWBRN37200MemDesc = NULL;
2820                 }
2821         }
2822
2823         RGXSetupFaultReadRegisterRollback(psDevInfo);
2824
2825         if (psDevInfo->psPowSyncPrim != NULL)
2826         {
2827                 SyncPrimFree(psDevInfo->psPowSyncPrim);
2828                 psDevInfo->psPowSyncPrim = NULL;
2829         }
2830
2831         if (psDevInfo->hSyncPrimContext != 0)
2832         {
2833                 SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
2834                 psDevInfo->hSyncPrimContext = 0;
2835         }
2836
2837 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
2838         if (psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc)
2839         {
2840                 if (psDevInfo->psRGXFWIfGpuUtilFWCb != NULL)
2841                 {
2842                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc);
2843                         psDevInfo->psRGXFWIfGpuUtilFWCb = NULL;
2844                 }
2845                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc);
2846                 psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc = NULL;
2847         }
2848
2849         RGXHWPerfDeinit();
2850
2851         if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
2852         {
2853                 if (psDevInfo->psRGXFWIfRuntimeCfg != NULL)
2854                 {
2855                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
2856                         psDevInfo->psRGXFWIfRuntimeCfg = NULL;
2857                 }
2858                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
2859                 psDevInfo->psRGXFWIfRuntimeCfgMemDesc = NULL;
2860         }
2861
2862         if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
2863         {
2864                 if (psDevInfo->psRGXFWIfHWRInfoBuf != NULL)
2865                 {
2866                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
2867                         psDevInfo->psRGXFWIfHWRInfoBuf = NULL;
2868                 }
2869                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
2870                 psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = NULL;
2871         }
2872
2873         if ((0 != psDevInfo->sDevFeatureCfg.ui32MCMS) && \
2874                                 (0 == (ui64ErnsBrns & FIX_HW_BRN_50767_BIT_MASK)))
2875         {
2876                 if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
2877                 {
2878                         DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
2879                         psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
2880                 }
2881         }
2882
2883         if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
2884         {
2885                 if (psDevInfo->psRGXFWIfTraceBuf != NULL)
2886                 {
2887                         /* first deinit/free the tracebuffer allocation */
2888                         RGXTraceBufferDeinit(psDevInfo);
2889
2890                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
2891                         psDevInfo->psRGXFWIfTraceBuf = NULL;
2892                 }
2893                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
2894                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc = NULL;
2895         }
2896 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
2897         if (psDevInfo->psRGXFWIfRegCfgMemDesc)
2898         {
2899                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfRegCfgMemDesc);
2900                 psDevInfo->psRGXFWIfRegCfgMemDesc = NULL;
2901         }
2902 #endif
2903         if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
2904         {
2905                 RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
2906                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
2907                 psDevInfo->psRGXFWIfHWPerfCountersMemDesc = NULL;
2908         }
2909         if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_SLC_VIVT_BIT_MASK)
2910         {
2911                 _FreeSLC3Fence(psDevInfo);
2912         }
2913
2914         if( (psDevInfo->sDevFeatureCfg.ui32META) && (psDevInfo->psMETAT1StackMemDesc))
2915         {
2916                 DevmemFwFree(psDevInfo, psDevInfo->psMETAT1StackMemDesc);
2917                 psDevInfo->psMETAT1StackMemDesc = NULL;
2918         }
2919
2920 #if defined(SUPPORT_PDVFS)
2921         if (psDevInfo->psRGXFWIFCoreClkRateMemDesc)
2922         {
2923                 if (psDevInfo->pui32RGXFWIFCoreClkRate != NULL)
2924                 {
2925                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc);
2926                         psDevInfo->pui32RGXFWIFCoreClkRate = NULL;
2927                 }
2928
2929                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIFCoreClkRateMemDesc);
2930                 psDevInfo->psRGXFWIFCoreClkRateMemDesc = NULL;
2931         }
2932 #endif
2933
2934         if (psDevInfo->psRGXFWIfInitMemDesc)
2935         {
2936                 DevmemFwFree(psDevInfo, psDevInfo->psRGXFWIfInitMemDesc);
2937                 psDevInfo->psRGXFWIfInitMemDesc = NULL;
2938         }
2939 #endif
2940
2941         if (psDevInfo->psCompletedMemDesc)
2942         {
2943                 if (psDevInfo->pui32CompletedById)
2944                 {
2945                         DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
2946                         psDevInfo->pui32CompletedById = NULL;
2947                 }
2948                 DevmemFwFree(psDevInfo, psDevInfo->psCompletedMemDesc);
2949                 psDevInfo->psCompletedMemDesc = NULL;
2950         }
2951         if (psDevInfo->psEndTimeMemDesc)
2952         {
2953                 if (psDevInfo->pui64EndTimeById)
2954                 {
2955                         DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
2956                         psDevInfo->pui64EndTimeById = NULL;
2957                 }
2958
2959                 DevmemFwFree(psDevInfo, psDevInfo->psEndTimeMemDesc);
2960                 psDevInfo->psEndTimeMemDesc = NULL;
2961         }
2962         if (psDevInfo->psStartTimeMemDesc)
2963         {
2964                 if (psDevInfo->pui64StartTimeById)
2965                 {
2966                         DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
2967                         psDevInfo->pui64StartTimeById = NULL;
2968                 }
2969
2970                 DevmemFwFree(psDevInfo, psDevInfo->psStartTimeMemDesc);
2971                 psDevInfo->psStartTimeMemDesc = NULL;
2972         }
2973 }
2974
2975
2976 /******************************************************************************
2977  FUNCTION       : RGXAcquireKernelCCBSlot
2978
2979  PURPOSE        : Attempts to obtain a slot in the Kernel CCB
2980
2981  PARAMETERS     : psCCB - the CCB
2982                         : Address of space if available, NULL otherwise
2983
2984  RETURNS        : PVRSRV_ERROR
2985 ******************************************************************************/
2986 static PVRSRV_ERROR RGXAcquireKernelCCBSlot(DEVMEM_MEMDESC *psKCCBCtrlMemDesc,
2987                                                                                         RGXFWIF_CCB_CTL *psKCCBCtl,
2988                                                                                         IMG_UINT32                      *pui32Offset)
2989 {
2990         IMG_UINT32      ui32OldWriteOffset, ui32NextWriteOffset;
2991
2992         ui32OldWriteOffset = psKCCBCtl->ui32WriteOffset;
2993         ui32NextWriteOffset = (ui32OldWriteOffset + 1) & psKCCBCtl->ui32WrapMask;
2994
2995         /* Note: The MTS can queue up to 255 kicks (254 pending kicks and 1 executing kick)
2996          * Hence the kernel CCB should not queue more 254 commands
2997          */
2998         PVR_ASSERT(psKCCBCtl->ui32WrapMask < 255);
2999
3000 #if defined(PDUMP)
3001         /* Wait for sufficient CCB space to become available */
3002         PDUMPCOMMENTWITHFLAGS(0, "Wait for kCCB woff=%u", ui32NextWriteOffset);
3003         DevmemPDumpCBP(psKCCBCtrlMemDesc,
3004                        offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
3005                        ui32NextWriteOffset,
3006                        1,
3007                        (psKCCBCtl->ui32WrapMask + 1));
3008 #endif
3009
3010         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3011         {
3012
3013                 if (ui32NextWriteOffset != psKCCBCtl->ui32ReadOffset)
3014                 {
3015                         *pui32Offset = ui32NextWriteOffset;
3016                         return PVRSRV_OK;
3017                 }
3018                 {
3019                         /*
3020                          * The following sanity check doesn't impact performance,
3021                          * since the CPU has to wait for the GPU anyway (full kernel CCB).
3022                          */
3023                         if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
3024                         {
3025                                 return PVRSRV_ERROR_KERNEL_CCB_FULL;
3026                         }
3027                 }
3028
3029                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3030         } END_LOOP_UNTIL_TIMEOUT();
3031
3032         /* Time out on waiting for CCB space */
3033         return PVRSRV_ERROR_KERNEL_CCB_FULL;
3034 }
3035
3036
3037 PVRSRV_ERROR RGXSendCommandWithPowLock(PVRSRV_RGXDEV_INFO       *psDevInfo,
3038                                                                                  RGXFWIF_DM                     eKCCBType,
3039                                                                                  RGXFWIF_KCCB_CMD       *psKCCBCmd,
3040                                                                                  IMG_UINT32                     ui32CmdSize,
3041                                                                                  IMG_UINT32                     ui32PDumpFlags)
3042 {
3043         PVRSRV_ERROR            eError;
3044         PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
3045
3046         /* Ensure Rogue is powered up before kicking MTS */
3047         eError = PVRSRVPowerLock(psDeviceNode);
3048
3049         if (eError != PVRSRV_OK)
3050         {
3051                 PVR_DPF((PVR_DBG_WARNING, "RGXSendCommandWithPowLock: failed to acquire powerlock (%s)",
3052                                         PVRSRVGetErrorStringKM(eError)));
3053
3054                 goto _PVRSRVPowerLock_Exit;
3055         }
3056
3057         PDUMPPOWCMDSTART();
3058         eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
3059                                                                                  PVRSRV_DEV_POWER_STATE_ON,
3060                                                                                  IMG_FALSE);
3061         PDUMPPOWCMDEND();
3062
3063         if (eError != PVRSRV_OK)
3064         {
3065                 PVR_DPF((PVR_DBG_WARNING, "RGXSendCommandWithPowLock: failed to transition Rogue to ON (%s)",
3066                                         PVRSRVGetErrorStringKM(eError)));
3067
3068                 goto _PVRSRVSetDevicePowerStateKM_Exit;
3069         }
3070
3071         eError = RGXSendCommand(psDevInfo, eKCCBType,  psKCCBCmd, ui32CmdSize, ui32PDumpFlags);
3072         if (eError != PVRSRV_OK)
3073         {
3074                 PVR_DPF((PVR_DBG_ERROR, "RGXSendCommandWithPowLock: failed to schedule command (%s)",
3075                                         PVRSRVGetErrorStringKM(eError)));
3076 #if defined(DEBUG)
3077                 /* PVRSRVDebugRequest must be called without powerlock */
3078                 PVRSRVPowerUnlock(psDeviceNode);
3079                 PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
3080                 goto _PVRSRVPowerLock_Exit;
3081 #endif
3082         }
3083
3084 _PVRSRVSetDevicePowerStateKM_Exit:
3085         PVRSRVPowerUnlock(psDeviceNode);
3086
3087 _PVRSRVPowerLock_Exit:
3088         return eError;
3089 }
3090
3091 static PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO        *psDevInfo,
3092                                                                  RGXFWIF_DM                     eKCCBType,
3093                                                                  RGXFWIF_KCCB_CMD       *psKCCBCmd,
3094                                                                  IMG_UINT32                     ui32CmdSize,
3095                                                                  IMG_UINT32             uiPdumpFlags)
3096 {
3097         PVRSRV_ERROR            eError;
3098         PVRSRV_DEVICE_NODE      *psDeviceNode = psDevInfo->psDeviceNode;
3099         RGXFWIF_CCB_CTL         *psKCCBCtl = psDevInfo->psKernelCCBCtl;
3100         IMG_UINT8                       *pui8KCCB = psDevInfo->psKernelCCB;
3101         IMG_UINT32                      ui32NewWriteOffset;
3102         IMG_UINT32                      ui32OldWriteOffset = psKCCBCtl->ui32WriteOffset;
3103
3104 #if !defined(PDUMP)
3105         PVR_UNREFERENCED_PARAMETER(uiPdumpFlags);
3106 #else
3107         IMG_BOOL bIsInCaptureRange;
3108         IMG_BOOL bPdumpEnabled;
3109         IMG_BOOL bPDumpPowTrans = PDUMPPOWCMDINTRANS();
3110
3111         PDumpIsCaptureFrameKM(&bIsInCaptureRange);
3112         bPdumpEnabled = (bIsInCaptureRange || PDUMP_IS_CONTINUOUS(uiPdumpFlags)) && !bPDumpPowTrans;
3113
3114         /* in capture range */
3115         if (bPdumpEnabled)
3116         {
3117                 if (!psDevInfo->bDumpedKCCBCtlAlready)
3118                 {
3119                         /* entering capture range */
3120                         psDevInfo->bDumpedKCCBCtlAlready = IMG_TRUE;
3121
3122                         /* wait for firmware to catch up */
3123                         PVR_DPF((PVR_DBG_MESSAGE, "RGXSendCommandRaw: waiting on fw to catch-up, roff: %d, woff: %d",
3124                                                 psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
3125                         PVRSRVPollForValueKM(&psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset, 0xFFFFFFFF);
3126
3127                         /* Dump Init state of Kernel CCB control (read and write offset) */
3128                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Initial state of kernel CCB Control, roff: %d, woff: %d",
3129                                                 psKCCBCtl->ui32ReadOffset, psKCCBCtl->ui32WriteOffset);
3130
3131                         DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
3132                                         0,
3133                                         sizeof(RGXFWIF_CCB_CTL),
3134                                         PDUMP_FLAGS_CONTINUOUS);
3135                 }
3136         }
3137 #endif
3138
3139         psKCCBCmd->eDM = eKCCBType;
3140
3141         PVR_ASSERT(ui32CmdSize == psKCCBCtl->ui32CmdSize);
3142         if (!OSLockIsLocked(psDeviceNode->hPowerLock))
3143         {
3144                 PVR_DPF((PVR_DBG_ERROR, "RGXSendCommandRaw called without power lock held!"));
3145                 PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
3146         }
3147
3148         /*
3149          * Acquire a slot in the CCB.
3150          */
3151         eError = RGXAcquireKernelCCBSlot(psDevInfo->psKernelCCBCtlMemDesc, psKCCBCtl, &ui32NewWriteOffset);
3152         if (eError != PVRSRV_OK)
3153         {
3154                 PVR_DPF((PVR_DBG_ERROR, "RGXSendCommandRaw failed to acquire CCB slot. Type:%u Error:%u",
3155                                 eKCCBType, eError));
3156                 goto _RGXSendCommandRaw_Exit;
3157         }
3158
3159         /*
3160          * Copy the command into the CCB.
3161          */
3162         OSDeviceMemCopy(&pui8KCCB[ui32OldWriteOffset * psKCCBCtl->ui32CmdSize],
3163                           psKCCBCmd, psKCCBCtl->ui32CmdSize);
3164
3165         /* ensure kCCB data is written before the offsets */
3166         OSWriteMemoryBarrier();
3167
3168         /* Move past the current command */
3169         psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
3170
3171
3172 #if defined(PDUMP)
3173         /* in capture range */
3174         if (bPdumpEnabled)
3175         {
3176                 /* Dump new Kernel CCB content */
3177                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Dump kCCB cmd for DM %d, woff = %d",
3178                                       eKCCBType,
3179                                       ui32OldWriteOffset);
3180                 DevmemPDumpLoadMem(psDevInfo->psKernelCCBMemDesc,
3181                                 ui32OldWriteOffset * psKCCBCtl->ui32CmdSize,
3182                                 psKCCBCtl->ui32CmdSize,
3183                                 PDUMP_FLAGS_CONTINUOUS);
3184
3185                 /* Dump new kernel CCB write offset */
3186                 PDUMPCOMMENTWITHFLAGS(uiPdumpFlags, "Dump kCCBCtl woff (added new cmd for DM %d): %d",
3187                                       eKCCBType,
3188                                       ui32NewWriteOffset);
3189                 DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
3190                                                            offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
3191                                                            sizeof(IMG_UINT32),
3192                                                            uiPdumpFlags);
3193         }
3194
3195         /* out of capture range */
3196         if (!bPdumpEnabled)
3197         {
3198                 RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset);
3199                 if (eError != PVRSRV_OK)
3200                 {
3201                         PVR_DPF((PVR_DBG_WARNING, "RGXSendCommandRaw: problem draining kCCB (%d)", eError));
3202                         goto _RGXSendCommandRaw_Exit;
3203                 }
3204         }
3205 #endif
3206
3207
3208         PDUMPCOMMENTWITHFLAGS(uiPdumpFlags, "MTS kick for kernel CCB");
3209         /*
3210          * Kick the MTS to schedule the firmware.
3211          */
3212     {
3213         IMG_UINT32 ui32MTSRegVal;
3214 #if defined(SUPPORT_PVRSRV_GPUVIRT)
3215         if(!(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK))
3216         {
3217                 ui32MTSRegVal = ((RGXFWIF_DM_GP + PVRSRV_GPUVIRT_OSID) & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_COUNTED;
3218         }else
3219 #endif
3220         {
3221             ui32MTSRegVal = (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_COUNTED;
3222         }
3223
3224
3225         __MTSScheduleWrite(psDevInfo, ui32MTSRegVal);
3226
3227         PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE, ui32MTSRegVal, uiPdumpFlags);
3228     }
3229
3230 #if defined (NO_HARDWARE)
3231         /* keep the roff updated because fw isn't there to update it */
3232         psKCCBCtl->ui32ReadOffset = psKCCBCtl->ui32WriteOffset;
3233 #endif
3234
3235 _RGXSendCommandRaw_Exit:
3236         return eError;
3237 }
3238
3239
3240 PVRSRV_ERROR RGXSendCommand(PVRSRV_RGXDEV_INFO  *psDevInfo,
3241                             RGXFWIF_DM          eKCCBType,
3242                             RGXFWIF_KCCB_CMD    *psKCCBCmd,
3243                             IMG_UINT32          ui32CmdSize,
3244                             IMG_UINT32          uiPdumpFlags)
3245 {
3246
3247         PVRSRV_ERROR eError = PVRSRV_OK;
3248         DLLIST_NODE *psNode, *psNext;
3249         RGX_DEFERRED_KCCB_CMD *psTempDeferredKCCBCmd;
3250
3251         /* Check if there is any deferred KCCB command before sending the command passed as argument */
3252         dllist_foreach_node(&psDevInfo->sKCCBDeferredCommandsListHead, psNode, psNext)
3253         {
3254                 psTempDeferredKCCBCmd = IMG_CONTAINER_OF(psNode, RGX_DEFERRED_KCCB_CMD, sListNode);
3255                 /* For every deferred KCCB command, try to send it*/
3256                 eError = RGXSendCommandRaw(psTempDeferredKCCBCmd->psDevInfo,
3257                                            psTempDeferredKCCBCmd->eDM,
3258                                            &(psTempDeferredKCCBCmd->sKCCBcmd),
3259                                            sizeof(psTempDeferredKCCBCmd->sKCCBcmd),
3260                                            psTempDeferredKCCBCmd->uiPdumpFlags);
3261                 if (eError != PVRSRV_OK)
3262                 {
3263                         goto _exit;
3264                 }
3265                 /* Remove from the deferred list the sent deferred KCCB command */
3266                 dllist_remove_node(psNode);
3267                 OSFreeMem(psTempDeferredKCCBCmd);
3268         }
3269
3270         eError = RGXSendCommandRaw(psDevInfo,
3271                                    eKCCBType,
3272                                    psKCCBCmd,
3273                                    ui32CmdSize,
3274                                    uiPdumpFlags);
3275
3276
3277 _exit:
3278         /*
3279          * If we don't manage to enqueue one of the deferred commands or the command
3280          * passed as argument because the KCCB is full, insert the latter into the deferred commands list.
3281          * The deferred commands will also be flushed eventually by:
3282          *  - one more KCCB command sent for any DM
3283          *  - the watchdog thread
3284          *  - the power off sequence
3285          */
3286         if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
3287         {
3288                 RGX_DEFERRED_KCCB_CMD *psDeferredCommand;
3289
3290                 psDeferredCommand = OSAllocMem(sizeof(*psDeferredCommand));
3291
3292                 if(!psDeferredCommand)
3293                 {
3294                         PVR_DPF((PVR_DBG_WARNING,"Deferring a KCCB command failed: allocation failure: requesting retry "));
3295                         eError = PVRSRV_ERROR_RETRY;
3296                 }
3297                 else
3298                 {
3299                         psDeferredCommand->sKCCBcmd = *psKCCBCmd;
3300                         psDeferredCommand->eDM = eKCCBType;
3301                         psDeferredCommand->uiPdumpFlags = uiPdumpFlags;
3302                         psDeferredCommand->psDevInfo = psDevInfo;
3303
3304                         PVR_DPF((PVR_DBG_WARNING,"Deferring a KCCB command for DM %d" ,eKCCBType));
3305                         dllist_add_to_tail(&(psDevInfo->sKCCBDeferredCommandsListHead), &(psDeferredCommand->sListNode));
3306
3307                         eError = PVRSRV_OK;
3308                 }
3309         }
3310
3311         return eError;
3312
3313 }
3314
3315
3316 void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
3317 {
3318         PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
3319         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
3320
3321         OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
3322 }
3323
3324 /*!
3325 ******************************************************************************
3326
3327  @Function      _RGXScheduleProcessQueuesMISR
3328
3329  @Description - Sends uncounted kick to all the DMs (the FW will process all
3330                                 the queue for all the DMs)
3331 ******************************************************************************/
3332 static void _RGXScheduleProcessQueuesMISR(void *pvData)
3333 {
3334         PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
3335         PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
3336         PVRSRV_ERROR           eError;
3337         PVRSRV_DEV_POWER_STATE ePowerState;
3338
3339         /* We don't need to acquire the BridgeLock as this power transition won't
3340            send a command to the FW */
3341         eError = PVRSRVPowerLock(psDeviceNode);
3342         if (eError != PVRSRV_OK)
3343         {
3344                 PVR_DPF((PVR_DBG_WARNING, "RGXScheduleProcessQueuesKM: failed to acquire powerlock (%s)",
3345                                         PVRSRVGetErrorStringKM(eError)));
3346
3347                 return;
3348         }
3349
3350         /* Check whether it's worth waking up the GPU */
3351         eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
3352
3353         if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
3354         {
3355 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
3356                 /* For now, guest drivers will always wake-up the GPU */
3357                 RGXFWIF_GPU_UTIL_FWCB  *psUtilFWCb = psDevInfo->psRGXFWIfGpuUtilFWCb;
3358                 IMG_BOOL               bGPUHasWorkWaiting;
3359
3360                 bGPUHasWorkWaiting =
3361                     (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFWCb->ui64LastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
3362
3363                 if (!bGPUHasWorkWaiting)
3364                 {
3365                         /* all queues are empty, don't wake up the GPU */
3366                         PVRSRVPowerUnlock(psDeviceNode);
3367                         return;
3368                 }
3369 #endif
3370         }
3371
3372         PDUMPPOWCMDSTART();
3373         /* wake up the GPU */
3374         eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
3375                                                                                  PVRSRV_DEV_POWER_STATE_ON,
3376                                                                                  IMG_FALSE);
3377         PDUMPPOWCMDEND();
3378
3379         if (eError != PVRSRV_OK)
3380         {
3381                 PVR_DPF((PVR_DBG_WARNING, "RGXScheduleProcessQueuesKM: failed to transition Rogue to ON (%s)",
3382                                         PVRSRVGetErrorStringKM(eError)));
3383
3384                 PVRSRVPowerUnlock(psDeviceNode);
3385                 return;
3386         }
3387
3388         /* uncounted kick to the FW */
3389         {
3390                 IMG_UINT32 ui32MTSRegVal;
3391 #if defined(SUPPORT_PVRSRV_GPUVIRT)
3392         if(!(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK))
3393         {
3394                         ui32MTSRegVal = ((RGXFWIF_DM_GP + PVRSRV_GPUVIRT_OSID) & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) |  RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED;
3395         }else
3396 #endif
3397         {
3398                 ui32MTSRegVal = (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED;
3399         }
3400
3401                 HTBLOGK(HTB_SF_MAIN_KICK_UNCOUNTED);
3402                 __MTSScheduleWrite(psDevInfo, ui32MTSRegVal);
3403         }
3404
3405         PVRSRVPowerUnlock(psDeviceNode);
3406 }
3407
3408 PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
3409 {
3410         return OSInstallMISR(phMISR,
3411                              _RGXScheduleProcessQueuesMISR,
3412                              psDeviceNode);
3413 }
3414
3415 /*!
3416 ******************************************************************************
3417
3418  @Function      RGXScheduleCommand
3419
3420  @Description - Submits a CCB command and kicks the firmware but first schedules
3421                 any commands which have to happen before handle
3422
3423  @Input psDevInfo                - pointer to device info
3424  @Input eKCCBType                - see RGXFWIF_CMD_*
3425  @Input psKCCBCmd                - kernel CCB command
3426  @Input ui32CmdSize      - kernel CCB SIZE
3427  @Input ui32CacheOpFence - CPU dcache operation fence
3428  @Input ui32PDumpFlags - PDUMP_FLAGS_CONTINUOUS bit set if the pdump flags should be continuous
3429
3430
3431  @Return PVRSRV_ERROR
3432
3433 ******************************************************************************/
3434 PVRSRV_ERROR RGXScheduleCommand(PVRSRV_RGXDEV_INFO      *psDevInfo,
3435                                                                 RGXFWIF_DM                      eKCCBType,
3436                                                                 RGXFWIF_KCCB_CMD        *psKCCBCmd,
3437                                                                 IMG_UINT32                      ui32CmdSize,
3438                                                                 IMG_UINT32                      ui32CacheOpFence,
3439                                                                 IMG_UINT32                      ui32PDumpFlags)
3440 {
3441         PVRSRV_ERROR eError;
3442         IMG_UINT32 uiMMUSyncUpdate;
3443
3444         eError = CacheOpFence(eKCCBType, ui32CacheOpFence);
3445         if (eError != PVRSRV_OK) goto RGXScheduleCommand_exit;
3446
3447 #if defined (SUPPORT_VALIDATION)
3448         /* For validation, force the core to different dust count states with each kick */
3449         if ((eKCCBType == RGXFWIF_DM_TA) || (eKCCBType == RGXFWIF_DM_CDM))
3450         {
3451                 if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_DUST_REQUEST_INJECT_EN)
3452                 {
3453                         IMG_UINT32 ui32NumDusts = RGXGetNextDustCount(&psDevInfo->sDustReqState, psDevInfo->sDevFeatureCfg.ui32MAXDustCount);
3454                         PVRSRVDeviceDustCountChange(psDevInfo->psDeviceNode, ui32NumDusts);
3455                 }
3456         }
3457 #endif
3458
3459         eError = RGXPreKickCacheCommand(psDevInfo, eKCCBType, &uiMMUSyncUpdate, IMG_FALSE);
3460         if (eError != PVRSRV_OK) goto RGXScheduleCommand_exit;
3461
3462         eError = RGXSendCommandWithPowLock(psDevInfo, eKCCBType, psKCCBCmd, ui32CmdSize, ui32PDumpFlags);
3463         if (eError != PVRSRV_OK) goto RGXScheduleCommand_exit;
3464
3465 RGXScheduleCommand_exit:
3466         return eError;
3467 }
3468
3469 /*
3470  * RGXCheckFirmwareCCB
3471  */
3472 void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
3473 {
3474         RGXFWIF_FWCCB_CMD *psFwCCBCmd;
3475
3476         RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
3477         IMG_UINT8 *psFWCCB = psDevInfo->psFirmwareCCB;
3478
3479         while (psFWCCBCtl->ui32ReadOffset != psFWCCBCtl->ui32WriteOffset)
3480         {
3481                 /* Point to the next command */
3482                 psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtl->ui32ReadOffset;
3483
3484                 HTBLOGK(HTB_SF_MAIN_FWCCB_CMD, psFwCCBCmd->eCmdType);
3485                 switch(psFwCCBCmd->eCmdType)
3486                 {
3487                         case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
3488                         {
3489                                 if (psDevInfo->bPDPEnabled)
3490                                 {
3491                                         PDUMP_PANIC(ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
3492                                 }
3493                                 RGXProcessRequestZSBufferBacking(psDevInfo,
3494                                         psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
3495                                 break;
3496                         }
3497
3498                         case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
3499                         {
3500                                 if (psDevInfo->bPDPEnabled)
3501                                 {
3502                                         PDUMP_PANIC(ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
3503                                 }
3504                                 RGXProcessRequestZSBufferUnbacking(psDevInfo,
3505                                         psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
3506                                 break;
3507                         }
3508
3509                         case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
3510                         {
3511                                 if (psDevInfo->bPDPEnabled)
3512                                 {
3513                                         PDUMP_PANIC(FREELIST_GROW, "Request to grow the free list");
3514                                 }
3515                                 RGXProcessRequestGrow(psDevInfo,
3516                                         psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
3517                                 break;
3518                         }
3519
3520                         case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
3521                         {
3522                                 if (psDevInfo->bPDPEnabled)
3523                                 {
3524                                         PDUMP_PANIC(FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
3525                                 }
3526 #if defined(PVRSRV_GPUVIRT_GUESTDRV)
3527                                 PVR_DPF((PVR_DBG_MESSAGE, "RGXCheckFirmwareCCBs: Freelist reconstruction request (%d) for %d freelists",
3528                                         psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
3529                                         psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
3530 #else
3531                                 PVR_DPF((PVR_DBG_MESSAGE, "RGXCheckFirmwareCCBs: Freelist reconstruction request (%d/%d) for %d freelists",
3532                                         psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
3533                                         psDevInfo->psRGXFWIfTraceBuf->ui32HwrCounter+1,
3534                                         psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
3535 #endif
3536
3537                                 RGXProcessRequestFreelistsReconstruction(psDevInfo,
3538                                         psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
3539                                         psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
3540                                 break;
3541                         }
3542
3543                         case RGXFWIF_FWCCB_CMD_DOPPLER_MEMORY_GROW:
3544                         {
3545                                 if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_RAY_TRACING_BIT_MASK)
3546                                 {
3547                                         if (psDevInfo->bPDPEnabled)
3548                                         {
3549                                                 PDUMP_PANIC(FREELIST_GROW, "Request to grow the RPM free list");
3550                                         }
3551                                         RGXProcessRequestRPMGrow(psDevInfo,
3552                                                         psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
3553                                 }
3554                         }
3555
3556                         case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
3557                         {
3558                                 DLLIST_NODE *psNode, *psNext;
3559                                 RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification =
3560                                         &psFwCCBCmd->uCmdData.sCmdContextResetNotification;
3561                                 IMG_UINT32 ui32ServerCommonContextID =
3562                                         psCmdContextResetNotification->ui32ServerCommonContextID;
3563                                 RGX_SERVER_COMMON_CONTEXT *psServerCommonContext = NULL;
3564
3565                                 dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
3566                                 {
3567                                         RGX_SERVER_COMMON_CONTEXT *psThisContext =
3568                                                 IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
3569
3570                                         if (psThisContext->ui32ContextID == ui32ServerCommonContextID)
3571                                         {
3572                                                 psServerCommonContext = psThisContext;
3573                                                 break;
3574                                         }
3575                                 }
3576
3577                                 PVR_DPF((PVR_DBG_MESSAGE, "RGXCheckFirmwareCCBs: Context 0x%p reset (ID=0x%08x, Reason=%d, JobRef=0x%08x)",
3578                                         psServerCommonContext,
3579                                         psCmdContextResetNotification->ui32ServerCommonContextID,
3580                                         (IMG_UINT32)(psCmdContextResetNotification->eResetReason),
3581                                         psCmdContextResetNotification->ui32ResetJobRef));
3582
3583                                 if (psServerCommonContext != NULL)
3584                                 {
3585                                         psServerCommonContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
3586                                         psServerCommonContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
3587                                 }
3588
3589                                 if (psCmdContextResetNotification->bPageFault)
3590                                 {
3591                                         DevmemIntPFNotify(psDevInfo->psDeviceNode,
3592                                                           psCmdContextResetNotification->ui64PCAddress);
3593                                 }
3594                                 break;
3595                         }
3596
3597                         case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
3598                         {
3599                                 RGXDumpDebugInfo(NULL,NULL,psDevInfo);
3600                                 break;
3601                         }
3602
3603                         case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
3604                         {
3605 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
3606                                 IMG_PID pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
3607                                 IMG_INT32 i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
3608
3609                                 switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
3610                                 {
3611                                         case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
3612                                         {
3613                                                 PVRSRVStatsUpdateRenderContextStats(i32AdjustmentValue,0,0,0,0,0,pidTmp);
3614                                                 break;
3615                                         }
3616                                         case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
3617                                         {
3618                                                 PVRSRVStatsUpdateRenderContextStats(0,i32AdjustmentValue,0,0,0,0,pidTmp);
3619                                                 break;
3620                                         }
3621                                         case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
3622                                         {
3623                                                 PVRSRVStatsUpdateRenderContextStats(0,0,i32AdjustmentValue,0,0,0,pidTmp);
3624                                                 break;
3625                                         }
3626                                         case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
3627                                         {
3628                                                 PVRSRVStatsUpdateRenderContextStats(0,0,0,i32AdjustmentValue,0,0,pidTmp);
3629                                                 break;
3630                                         }
3631                                         case RGXFWIF_FWCCB_CMD_UPDATE_NUM_SH_STORES:
3632                                         {
3633                                                 PVRSRVStatsUpdateRenderContextStats(0,0,0,0,i32AdjustmentValue,0,pidTmp);
3634                                                 break;
3635                                         }
3636                                         case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
3637                                         {
3638                                                 PVRSRVStatsUpdateRenderContextStats(0,0,0,0,0,i32AdjustmentValue,pidTmp);
3639                                                 break;
3640                                         }
3641                                 }
3642 #endif
3643                                 break;
3644                         }
3645                         case RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE:
3646                         {
3647 #if defined(SUPPORT_PDVFS)
3648                                 PDVFSProcessCoreClkRateChange(psDevInfo,
3649                                                                                           psFwCCBCmd->uCmdData.sCmdCoreClkRateChange.ui32CoreClkRate);
3650 #endif
3651                                 break;
3652                         }
3653                         default:
3654                         {
3655                                 PVR_ASSERT(IMG_FALSE);
3656                         }
3657                 }
3658
3659                 /* Update read offset */
3660                 psFWCCBCtl->ui32ReadOffset = (psFWCCBCtl->ui32ReadOffset + 1) & psFWCCBCtl->ui32WrapMask;
3661         }
3662 }
3663
3664 /*
3665  * PVRSRVRGXFrameworkCopyCommand
3666  */
3667 PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(DEVMEM_MEMDESC       *psFWFrameworkMemDesc,
3668                                                                                    IMG_PBYTE            pbyGPUFRegisterList,
3669                                                                                    IMG_UINT32           ui32FrameworkRegisterSize)
3670 {
3671         PVRSRV_ERROR    eError;
3672         RGXFWIF_RF_REGISTERS    *psRFReg;
3673
3674         eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
3675                                       (void **)&psRFReg);
3676         if (eError != PVRSRV_OK)
3677         {
3678                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXFrameworkCopyCommand: Failed to map firmware render context state (%u)",
3679                                 eError));
3680                 return eError;
3681         }
3682
3683         OSDeviceMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
3684
3685         /* Release the CPU mapping */
3686         DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
3687
3688         /*
3689          * Dump the FW framework buffer
3690          */
3691         PDUMPCOMMENT("Dump FWFramework buffer");
3692         DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
3693
3694         return PVRSRV_OK;
3695 }
3696
3697 /*
3698  * PVRSRVRGXFrameworkCreateKM
3699  */
3700 PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE      *psDeviceNode,
3701                                                                                 DEVMEM_MEMDESC          **ppsFWFrameworkMemDesc,
3702                                                                                 IMG_UINT32                      ui32FrameworkCommandSize)
3703 {
3704         PVRSRV_ERROR                    eError;
3705         PVRSRV_RGXDEV_INFO              *psDevInfo = psDeviceNode->pvDevice;
3706
3707         /*
3708                 Allocate device memory for the firmware GPU framework state.
3709                 Sufficient info to kick one or more DMs should be contained in this buffer
3710         */
3711         PDUMPCOMMENT("Allocate Rogue firmware framework state");
3712
3713         eError = DevmemFwAllocate(psDevInfo,
3714                                                           ui32FrameworkCommandSize,
3715                                                           RGX_FWCOMCTX_ALLOCFLAGS,
3716                                                           "FwGPUFrameworkState",
3717                                                           ppsFWFrameworkMemDesc);
3718
3719         if (eError != PVRSRV_OK)
3720         {
3721                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXFrameworkContextKM: Failed to allocate firmware framework state (%u)",
3722                                 eError));
3723                 return eError;
3724         }
3725
3726         return PVRSRV_OK;
3727 }
3728
3729 PVRSRV_ERROR RGXWaitForFWOp(PVRSRV_RGXDEV_INFO  *psDevInfo,
3730                             RGXFWIF_DM eDM,
3731                             PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim,
3732                             IMG_UINT32 ui32PDumpFlags)
3733 {
3734         PVRSRV_ERROR            eError = PVRSRV_OK;
3735         PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
3736         RGXFWIF_KCCB_CMD        sCmdSyncPrim;
3737
3738         /* Ensure Rogue is powered up before kicking MTS */
3739         eError = PVRSRVPowerLock(psDeviceNode);
3740
3741         if (eError != PVRSRV_OK)
3742         {
3743                 PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire powerlock (%s)",
3744                                         __FUNCTION__,
3745                                         PVRSRVGetErrorStringKM(eError)));
3746
3747                 goto _PVRSRVPowerLock_Exit;
3748         }
3749
3750         PDUMPPOWCMDSTART();
3751         eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
3752                                              PVRSRV_DEV_POWER_STATE_ON,
3753                                              IMG_FALSE);
3754         PDUMPPOWCMDEND();
3755
3756         if (eError != PVRSRV_OK)
3757         {
3758                 PVR_DPF((PVR_DBG_ERROR, "%s: failed to transition Rogue to ON (%s)",
3759                                         __FUNCTION__,
3760                                         PVRSRVGetErrorStringKM(eError)));
3761
3762                 goto _PVRSRVSetDevicePowerStateKM_Exit;
3763         }
3764
3765         /* Setup sync primitive */
3766         eError = SyncPrimSet(psSyncPrim, 0);
3767         if (eError != PVRSRV_OK)
3768         {
3769                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to set SyncPrim (%u)",
3770                         __FUNCTION__, eError));
3771                 goto _SyncPrimSet_Exit;
3772         }
3773
3774         /* prepare a sync command */
3775         eError = SyncPrimGetFirmwareAddr(psSyncPrim,
3776                         &sCmdSyncPrim.uCmdData.sSyncData.sSyncObjDevVAddr.ui32Addr);
3777         if (eError != PVRSRV_OK)
3778         {
3779                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to get SyncPrim FW address(%u)",
3780                         __FUNCTION__, eError));
3781                 goto _SyncPrimGetFirmwareAddr_Exit;
3782         }
3783         sCmdSyncPrim.eCmdType = RGXFWIF_KCCB_CMD_SYNC;
3784         sCmdSyncPrim.uCmdData.sSyncData.uiUpdateVal = 1;
3785
3786         PDUMPCOMMENT("RGXWaitForFWOp: Submit Kernel SyncPrim [0x%08x] to DM %d ",
3787                 sCmdSyncPrim.uCmdData.sSyncData.sSyncObjDevVAddr.ui32Addr, eDM);
3788
3789         /* submit the sync primitive to the kernel CCB */
3790         eError = RGXSendCommand(psDevInfo,
3791                                 eDM,
3792                                 &sCmdSyncPrim,
3793                                 sizeof(RGXFWIF_KCCB_CMD),
3794                                 ui32PDumpFlags);
3795         if (eError != PVRSRV_OK)
3796         {
3797                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to schedule Kernel SyncPrim with error (%u)",
3798                                         __FUNCTION__,
3799                                         eError));
3800                 goto _RGXSendCommandRaw_Exit;
3801         }
3802
3803         /* Wait for sync primitive to be updated */
3804 #if defined(PDUMP)
3805         PDUMPCOMMENT("RGXScheduleCommandAndWait: Poll for Kernel SyncPrim [0x%08x] on DM %d ",
3806                 sCmdSyncPrim.uCmdData.sSyncData.sSyncObjDevVAddr.ui32Addr, eDM);
3807
3808         SyncPrimPDumpPol(psSyncPrim,
3809                          1,
3810                          0xffffffff,
3811                          PDUMP_POLL_OPERATOR_EQUAL,
3812                          ui32PDumpFlags);
3813 #endif
3814
3815         {
3816                 RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
3817                 IMG_UINT32 ui32CurrentQueueLength =
3818                                 (psKCCBCtl->ui32WrapMask+1 +
3819                                 psKCCBCtl->ui32WriteOffset -
3820                                 psKCCBCtl->ui32ReadOffset) & psKCCBCtl->ui32WrapMask;
3821                 IMG_UINT32 ui32MaxRetries;
3822
3823                 for (ui32MaxRetries = (ui32CurrentQueueLength + 1) * 3;
3824                          ui32MaxRetries > 0;
3825                          ui32MaxRetries--)
3826                 {
3827                         eError = PVRSRVWaitForValueKMAndHoldBridgeLockKM(psSyncPrim->pui32LinAddr, 1, 0xffffffff);
3828
3829                         if (eError != PVRSRV_ERROR_TIMEOUT)
3830                         {
3831                                 break;
3832                         }
3833                 }
3834
3835                 if (eError == PVRSRV_ERROR_TIMEOUT)
3836                 {
3837                         PVR_DPF((PVR_DBG_ERROR,"%s: PVRSRVWaitForValueKMAndHoldBridgeLock timed out. Dump debug information.",
3838                                         __FUNCTION__));
3839                         PVRSRVPowerUnlock(psDeviceNode);
3840
3841                         PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
3842                         PVR_ASSERT(eError != PVRSRV_ERROR_TIMEOUT);
3843                         goto _PVRSRVDebugRequest_Exit;
3844                 }
3845         }
3846
3847 _RGXSendCommandRaw_Exit:
3848 _SyncPrimGetFirmwareAddr_Exit:
3849 _SyncPrimSet_Exit:
3850 _PVRSRVSetDevicePowerStateKM_Exit:
3851
3852         PVRSRVPowerUnlock(psDeviceNode);
3853
3854 _PVRSRVDebugRequest_Exit:
3855 _PVRSRVPowerLock_Exit:
3856         return eError;
3857 }
3858
3859 PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
3860                                 IMG_UINT32 ui32Config,
3861                                 IMG_UINT32 *pui32ConfigState,
3862                                 IMG_BOOL bSetNotClear)
3863 {
3864         PVRSRV_ERROR eError;
3865         RGXFWIF_KCCB_CMD sStateFlagCmd;
3866         PVRSRV_CLIENT_SYNC_PRIM *psResponseSync;
3867
3868         if (!psDevInfo)
3869         {
3870                 eError = PVRSRV_ERROR_INVALID_PARAMS;
3871                 goto return_;
3872         }
3873
3874         if (psDevInfo->psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
3875         {
3876                 eError = PVRSRV_ERROR_NOT_INITIALISED;
3877                 goto return_;
3878         }
3879
3880         sStateFlagCmd.eCmdType = RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL;
3881         sStateFlagCmd.eDM = RGXFWIF_DM_GP;
3882         sStateFlagCmd.uCmdData.sStateFlagCtrl.ui32Config = ui32Config;
3883         sStateFlagCmd.uCmdData.sStateFlagCtrl.bSetNotClear = bSetNotClear;
3884
3885         eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psResponseSync, "rgx config flags");
3886         if (PVRSRV_OK != eError)
3887         {
3888                 goto return_;
3889         }
3890         eError = SyncPrimSet(psResponseSync, 0);
3891         if (eError != PVRSRV_OK)
3892         {
3893                 goto return_freesync_;
3894         }
3895
3896         eError = SyncPrimGetFirmwareAddr(psResponseSync, &sStateFlagCmd.uCmdData.sStateFlagCtrl.sSyncObjDevVAddr.ui32Addr);
3897         if (PVRSRV_OK != eError)
3898         {
3899                 goto return_freesync_;
3900         }
3901
3902         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3903         {
3904                 eError = RGXScheduleCommand(psDevInfo,
3905                                         RGXFWIF_DM_GP,
3906                                         &sStateFlagCmd,
3907                                         sizeof(sStateFlagCmd),
3908                                         0,
3909                                         PDUMP_FLAGS_CONTINUOUS);
3910                 if (eError != PVRSRV_ERROR_RETRY)
3911                 {
3912                         break;
3913                 }
3914                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3915         } END_LOOP_UNTIL_TIMEOUT();
3916         PVR_LOGG_IF_ERROR(eError, "RGXScheduleCommand", return_);
3917
3918         /* Wait for FW to complete */
3919         eError = RGXWaitForFWOp(psDevInfo,
3920                                 RGXFWIF_DM_GP,
3921                                 psDevInfo->psDeviceNode->psSyncPrim,
3922                                 PDUMP_FLAGS_CONTINUOUS);
3923         PVR_LOGG_IF_ERROR(eError, "RGXWaitForFWOp", return_);
3924
3925         if (pui32ConfigState)
3926         {
3927                 *pui32ConfigState = *psResponseSync->pui32LinAddr;
3928         }
3929
3930 return_freesync_:
3931         SyncPrimFree(psResponseSync);
3932 return_:
3933         return eError;
3934 }
3935
3936 static
3937 PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO       *psDevInfo,
3938                                                                            RGXFWIF_DM                   eDM,
3939                                                                            RGXFWIF_KCCB_CMD             *psKCCBCmd,
3940                                                                            IMG_UINT32                   ui32CmdSize,
3941                                                                            RGXFWIF_CLEANUP_TYPE eCleanupType,
3942                                                                            PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim,
3943                                                                            IMG_UINT32                           ui32PDumpFlags)
3944 {
3945         PVRSRV_ERROR eError;
3946
3947         psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
3948
3949         psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
3950         eError = SyncPrimGetFirmwareAddr(psSyncPrim, &psKCCBCmd->uCmdData.sCleanupData.sSyncObjDevVAddr.ui32Addr);
3951         if (eError != PVRSRV_OK)
3952         {
3953                 goto fail_command;
3954         }
3955
3956         eError = SyncPrimSet(psSyncPrim, 0);
3957         if (eError != PVRSRV_OK)
3958         {
3959                 goto fail_command;
3960         }
3961
3962         /*
3963                 Send the cleanup request to the firmware. If the resource is still busy
3964                 the firmware will tell us and we'll drop out with a retry.
3965         */
3966         eError = RGXScheduleCommand(psDevInfo,
3967                                                                 eDM,
3968                                                                 psKCCBCmd,
3969                                                                 ui32CmdSize,
3970                                                                 0,
3971                                                                 ui32PDumpFlags);
3972         if (eError != PVRSRV_OK)
3973         {
3974                 goto fail_command;
3975         }
3976
3977         /* Wait for sync primitive to be updated */
3978 #if defined(PDUMP)
3979         PDUMPCOMMENT("Wait for the firmware to reply to the cleanup command");
3980         SyncPrimPDumpPol(psSyncPrim,
3981                                         RGXFWIF_CLEANUP_RUN,
3982                                         RGXFWIF_CLEANUP_RUN,
3983                                         PDUMP_POLL_OPERATOR_EQUAL,
3984                                         ui32PDumpFlags);
3985
3986         /*
3987          * The cleanup request to the firmware will tell us if a given resource is busy or not.
3988          * If the RGXFWIF_CLEANUP_BUSY flag is set, this means that the resource is still in use.
3989          * In this case we return a PVRSRV_ERROR_RETRY error to the client drivers and they will
3990          * re-issue the cleanup request until it succeed.
3991          *
3992          * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
3993          * that cleanup requests are only submitted if the resource is unused.
3994          * If this is not the case, the following poll will block infinitely, making sure
3995          * the issue doesn't go unnoticed.
3996          */
3997         PDUMPCOMMENT("Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
3998                                         eDM,
3999                                         psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
4000                                         psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
4001         SyncPrimPDumpPol(psSyncPrim,
4002                                         0,
4003                                         RGXFWIF_CLEANUP_BUSY,
4004                                         PDUMP_POLL_OPERATOR_EQUAL,
4005                                         ui32PDumpFlags);
4006 #endif
4007
4008         {
4009                 RGXFWIF_CCB_CTL  *psKCCBCtl = psDevInfo->psKernelCCBCtl;
4010                 IMG_UINT32       ui32CurrentQueueLength = (psKCCBCtl->ui32WrapMask+1 +
4011                                                            psKCCBCtl->ui32WriteOffset -
4012                                                            psKCCBCtl->ui32ReadOffset) & psKCCBCtl->ui32WrapMask;
4013                 IMG_UINT32       ui32MaxRetries;
4014
4015                 for (ui32MaxRetries = ui32CurrentQueueLength + 1;
4016                          ui32MaxRetries > 0;
4017                          ui32MaxRetries--)
4018                 {
4019                         eError = PVRSRVWaitForValueKMAndHoldBridgeLockKM(psSyncPrim->pui32LinAddr, RGXFWIF_CLEANUP_RUN, RGXFWIF_CLEANUP_RUN);
4020
4021                         if (eError != PVRSRV_ERROR_TIMEOUT)
4022                         {
4023                                 break;
4024                         }
4025                 }
4026
4027                 /*
4028                         If the firmware hasn't got back to us in a timely manner
4029                         then bail and let the caller retry the command.
4030                 */
4031                 if (eError == PVRSRV_ERROR_TIMEOUT)
4032                 {
4033                         PVR_DPF((PVR_DBG_WARNING,"RGXScheduleCleanupCommand: PVRSRVWaitForValueKMAndHoldBridgeLock timed out. Dump debug information."));
4034
4035                         eError = PVRSRV_ERROR_RETRY;
4036 #if defined(DEBUG)
4037                         PVRSRVDebugRequest(psDevInfo->psDeviceNode,
4038                                                            DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
4039 #endif
4040                         goto fail_poll;
4041                 }
4042                 else if (eError != PVRSRV_OK)
4043                 {
4044                         goto fail_poll;
4045                 }
4046         }
4047
4048         /*
4049                 If the command has was run but a resource was busy, then the request
4050                 will need to be retried.
4051         */
4052         if (*psSyncPrim->pui32LinAddr & RGXFWIF_CLEANUP_BUSY)
4053         {
4054                 eError = PVRSRV_ERROR_RETRY;
4055                 goto fail_requestbusy;
4056         }
4057
4058         return PVRSRV_OK;
4059
4060 fail_requestbusy:
4061 fail_poll:
4062 fail_command:
4063         PVR_ASSERT(eError != PVRSRV_OK);
4064
4065         return eError;
4066 }
4067
4068 /*
4069         RGXRequestCommonContextCleanUp
4070 */
4071 PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
4072                                                                                           RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
4073                                                                                           PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim,
4074                                                                                           RGXFWIF_DM eDM,
4075                                                                                           IMG_UINT32 ui32PDumpFlags)
4076 {
4077         RGXFWIF_KCCB_CMD                        sRCCleanUpCmd = {0};
4078         PVRSRV_ERROR                            eError;
4079         PRGXFWIF_FWCOMMONCONTEXT        psFWCommonContextFWAddr;
4080
4081         psFWCommonContextFWAddr = FWCommonContextGetFWAddress(psServerCommonContext);
4082
4083         PDUMPCOMMENT("Common ctx cleanup Request DM%d [context = 0x%08x]",
4084                                         eDM, psFWCommonContextFWAddr.ui32Addr);
4085         PDUMPCOMMENT("Wait for CCB to be empty before common ctx cleanup");
4086
4087         RGXCCBPDumpDrainCCB(FWCommonContextGetClientCCB(psServerCommonContext), ui32PDumpFlags);
4088
4089         /* Setup our command data, the cleanup call will fill in the rest */
4090         sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
4091
4092         /* Request cleanup of the firmware resource */
4093         eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
4094                                                                            eDM,
4095                                                                            &sRCCleanUpCmd,
4096                                                                            sizeof(RGXFWIF_KCCB_CMD),
4097                                                                            RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
4098                                                                            psSyncPrim,
4099                                                                            ui32PDumpFlags);
4100
4101         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
4102         {
4103                 PVR_DPF((PVR_DBG_ERROR,"RGXRequestCommonContextCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
4104         }
4105
4106         return eError;
4107 }
4108
4109 /*
4110  * RGXRequestHWRTDataCleanUp
4111  */
4112
4113 PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
4114                                                                                  PRGXFWIF_HWRTDATA psHWRTData,
4115                                                                                  PVRSRV_CLIENT_SYNC_PRIM *psSync,
4116                                                                                  RGXFWIF_DM eDM)
4117 {
4118         RGXFWIF_KCCB_CMD                        sHWRTDataCleanUpCmd = {0};
4119         PVRSRV_ERROR                            eError;
4120
4121         PDUMPCOMMENT("HW RTData cleanup Request DM%d [HWRTData = 0x%08x]", eDM, psHWRTData.ui32Addr);
4122
4123         sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
4124
4125         eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
4126                                                                            eDM,
4127                                                                            &sHWRTDataCleanUpCmd,
4128                                                                            sizeof(sHWRTDataCleanUpCmd),
4129                                                                            RGXFWIF_CLEANUP_HWRTDATA,
4130                                                                            psSync,
4131                                                                            IMG_FALSE);
4132
4133         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
4134         {
4135                 PVR_DPF((PVR_DBG_ERROR,"RGXRequestHWRTDataCleanUp: Failed to schedule a HWRTData cleanup with error (%u)", eError));
4136         }
4137
4138         return eError;
4139 }
4140
4141 /*
4142         RGXFWRequestFreeListCleanUp
4143 */
4144 PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
4145                                                                                  PRGXFWIF_FREELIST psFWFreeList,
4146                                                                                  PVRSRV_CLIENT_SYNC_PRIM *psSync)
4147 {
4148         RGXFWIF_KCCB_CMD                        sFLCleanUpCmd = {0};
4149         PVRSRV_ERROR                            eError;
4150
4151         PDUMPCOMMENT("Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
4152
4153         /* Setup our command data, the cleanup call will fill in the rest */
4154         sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
4155
4156         /* Request cleanup of the firmware resource */
4157         eError = RGXScheduleCleanupCommand(psDevInfo,
4158                                                                            RGXFWIF_DM_GP,
4159                                                                            &sFLCleanUpCmd,
4160                                                                            sizeof(RGXFWIF_KCCB_CMD),
4161                                                                            RGXFWIF_CLEANUP_FREELIST,
4162                                                                            psSync,
4163                                                                            IMG_FALSE);
4164
4165         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
4166         {
4167                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestFreeListCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
4168         }
4169
4170         return eError;
4171 }
4172
4173 /*
4174         RGXFWRequestZSBufferCleanUp
4175 */
4176 PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
4177                                                                                  PRGXFWIF_ZSBUFFER psFWZSBuffer,
4178                                                                                  PVRSRV_CLIENT_SYNC_PRIM *psSync)
4179 {
4180         RGXFWIF_KCCB_CMD                        sZSBufferCleanUpCmd = {0};
4181         PVRSRV_ERROR                            eError;
4182
4183         PDUMPCOMMENT("ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
4184
4185         /* Setup our command data, the cleanup call will fill in the rest */
4186         sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
4187
4188         /* Request cleanup of the firmware resource */
4189         eError = RGXScheduleCleanupCommand(psDevInfo,
4190                                                                            RGXFWIF_DM_3D,
4191                                                                            &sZSBufferCleanUpCmd,
4192                                                                            sizeof(RGXFWIF_KCCB_CMD),
4193                                                                            RGXFWIF_CLEANUP_ZSBUFFER,
4194                                                                            psSync,
4195                                                                            IMG_FALSE);
4196
4197         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
4198         {
4199                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestZSBufferCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
4200         }
4201
4202         return eError;
4203 }
4204
4205
4206 PVRSRV_ERROR RGXFWRequestRayFrameDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
4207                                                                                          PRGXFWIF_RAY_FRAME_DATA psHWFrameData,
4208                                                                                          PVRSRV_CLIENT_SYNC_PRIM *psSync,
4209                                                                                          RGXFWIF_DM eDM)
4210 {
4211         RGXFWIF_KCCB_CMD                        sHWFrameDataCleanUpCmd = {0};
4212         PVRSRV_ERROR                            eError;
4213
4214         PDUMPCOMMENT("HW FrameData cleanup Request DM%d [HWFrameData = 0x%08x]", eDM, psHWFrameData.ui32Addr);
4215
4216         sHWFrameDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWFrameData = psHWFrameData;
4217
4218         eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
4219                                                                            eDM,
4220                                                                            &sHWFrameDataCleanUpCmd,
4221                                                                            sizeof(sHWFrameDataCleanUpCmd),
4222                                                                            RGXFWIF_CLEANUP_HWFRAMEDATA,
4223                                                                            psSync,
4224                                                                            IMG_FALSE);
4225
4226         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
4227         {
4228                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestRayFrameDataCleanUp: Failed to schedule a HWFrameData cleanup with error (%u)", eError));
4229         }
4230
4231         return eError;
4232 }
4233
4234 /*
4235         RGXFWRequestRPMFreeListCleanUp
4236 */
4237 PVRSRV_ERROR RGXFWRequestRPMFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
4238                                                                                         PRGXFWIF_RPM_FREELIST psFWRPMFreeList,
4239                                                                                         PVRSRV_CLIENT_SYNC_PRIM *psSync)
4240 {
4241         RGXFWIF_KCCB_CMD                        sFLCleanUpCmd = {0};
4242         PVRSRV_ERROR                            eError;
4243
4244         PDUMPCOMMENT("RPM Free list cleanup Request [RPM FreeList = 0x%08x]", psFWRPMFreeList.ui32Addr);
4245
4246         /* Setup our command data, the cleanup call will fill in the rest */
4247         sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psRPMFreelist = psFWRPMFreeList;
4248
4249         /* Request cleanup of the firmware resource */
4250         eError = RGXScheduleCleanupCommand(psDevInfo,
4251                                                                            RGXFWIF_DM_GP,
4252                                                                            &sFLCleanUpCmd,
4253                                                                            sizeof(RGXFWIF_KCCB_CMD),
4254                                                                            RGXFWIF_CLEANUP_RPM_FREELIST,
4255                                                                            psSync,
4256                                                                            IMG_FALSE);
4257
4258         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
4259         {
4260                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestRPMFreeListCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
4261         }
4262
4263         return eError;
4264 }
4265
4266 PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
4267                                                                 IMG_UINT32 ui32HCSDeadlineMs)
4268 {
4269         PVRSRV_ERROR eError;
4270         RGXFWIF_KCCB_CMD        sSetHCSDeadline;
4271
4272         sSetHCSDeadline.eCmdType                            = RGXFWIF_KCCB_CMD_HCS_SET_DEADLINE;
4273         sSetHCSDeadline.eDM                                 = RGXFWIF_DM_GP;
4274         sSetHCSDeadline.uCmdData.sHCSCtrl.ui32HCSDeadlineMS = ui32HCSDeadlineMs;
4275
4276         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4277         {
4278                 eError = RGXScheduleCommand(psDevInfo,
4279                                                                         RGXFWIF_DM_GP,
4280                                                                         &sSetHCSDeadline,
4281                                                                         sizeof(sSetHCSDeadline),
4282                                                                         0,
4283                                                                         PDUMP_FLAGS_CONTINUOUS);
4284
4285                 if (eError != PVRSRV_ERROR_RETRY)
4286                 {
4287                         break;
4288                 }
4289                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4290         } END_LOOP_UNTIL_TIMEOUT();
4291
4292         return eError;
4293 }
4294
4295 PVRSRV_ERROR RGXFWOSConfig(PVRSRV_RGXDEV_INFO *psDevInfo)
4296 {
4297         PVRSRV_ERROR eError;
4298         RGXFWIF_KCCB_CMD   sOSConfigCmdData;
4299
4300         sOSConfigCmdData.eCmdType                            = RGXFWIF_KCCB_CMD_OS_CFG_INIT;
4301         sOSConfigCmdData.eDM                                 = RGXFWIF_DM_GP;
4302         sOSConfigCmdData.uCmdData.sCmdOSConfigData.sOSInit   = psDevInfo->sFWInitFWAddr;
4303
4304         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4305         {
4306                 eError = RGXScheduleCommand(psDevInfo,
4307                                                                         RGXFWIF_DM_GP,
4308                                                                         &sOSConfigCmdData,
4309                                                                         sizeof(sOSConfigCmdData),
4310                                                                         0,
4311                                                                         PDUMP_FLAGS_CONTINUOUS);
4312                 if (eError != PVRSRV_ERROR_RETRY)
4313                 {
4314                         break;
4315                 }
4316                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4317         } END_LOOP_UNTIL_TIMEOUT();
4318
4319         return eError;
4320 }
4321
4322 PVRSRV_ERROR RGXFWSetOSIsolationThreshold(PVRSRV_RGXDEV_INFO *psDevInfo,
4323                                                                 IMG_UINT32 ui32IsolationPriorityThreshold)
4324 {
4325         PVRSRV_ERROR eError;
4326         RGXFWIF_KCCB_CMD        sOSidIsoConfCmd;
4327
4328         sOSidIsoConfCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ISOLATION_GROUP_CHANGE;
4329         sOSidIsoConfCmd.uCmdData.sCmdOSidIsolationData.ui32IsolationPriorityThreshold = ui32IsolationPriorityThreshold;
4330
4331         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4332         {
4333                 eError = RGXScheduleCommand(psDevInfo,
4334                                                                         RGXFWIF_DM_GP,
4335                                                                         &sOSidIsoConfCmd,
4336                                                                         sizeof(sOSidIsoConfCmd),
4337                                                                         0,
4338                                                                         PDUMP_FLAGS_CONTINUOUS);
4339                 if (eError != PVRSRV_ERROR_RETRY)
4340                 {
4341                         break;
4342                 }
4343                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4344         } END_LOOP_UNTIL_TIMEOUT();
4345
4346         return eError;
4347 }
4348
4349 PVRSRV_ERROR RGXFWSetVMOnlineState(PVRSRV_RGXDEV_INFO *psDevInfo,
4350                                                                 IMG_UINT32 ui32OSid,
4351                                                                 RGXFWIF_OS_STATE_CHANGE eOSOnlineState)
4352 {
4353         PVRSRV_ERROR         eError = PVRSRV_OK;
4354 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
4355         RGXFWIF_KCCB_CMD     sOSOnlineStateCmd;
4356         RGXFWIF_TRACEBUF    *psRGXFWIfTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
4357         volatile IMG_UINT32 *pui32OSStateFlags;
4358
4359         sOSOnlineStateCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE;
4360         sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.ui32OSid = ui32OSid;
4361         sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = eOSOnlineState;
4362
4363         if (eOSOnlineState == RGXFWIF_OS_ONLINE)
4364         {
4365                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4366                 {
4367                         eError = RGXScheduleCommand(psDevInfo,
4368                                                                                 RGXFWIF_DM_GP,
4369                                                                                 &sOSOnlineStateCmd,
4370                                                                                 sizeof(sOSOnlineStateCmd),
4371                                                                                 0,
4372                                                                                 PDUMP_FLAGS_CONTINUOUS);
4373                         if (eError != PVRSRV_ERROR_RETRY)
4374                         {
4375                                 break;
4376                         }
4377                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4378                 } END_LOOP_UNTIL_TIMEOUT();
4379
4380                 return eError;
4381         }
4382
4383         if (psRGXFWIfTraceBuf == NULL)
4384         {
4385                 return PVRSRV_ERROR_NOT_INITIALISED;
4386         }
4387         pui32OSStateFlags = (volatile IMG_UINT32*) &psRGXFWIfTraceBuf->ui32OSStateFlags[ui32OSid];
4388
4389         /* Attempt several times until the FW manages to offload the OS */
4390         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4391         {
4392                 IMG_UINT32 ui32OSStateFlags;
4393
4394                 /* Send request */
4395                 eError = RGXScheduleCommand(psDevInfo,
4396                                                                         RGXFWIF_DM_GP,
4397                                                                         &sOSOnlineStateCmd,
4398                                                                         sizeof(sOSOnlineStateCmd),
4399                                                                         0,
4400                                                                         IMG_TRUE);
4401                 if (unlikely(eError == PVRSRV_ERROR_RETRY))
4402                 {
4403                         continue;
4404                 }
4405                 PVR_LOGG_IF_ERROR(eError, "RGXScheduleCommand", return_);
4406
4407                 /* Wait for FW to process the cmd */
4408                 eError = RGXWaitForFWOp(psDevInfo,
4409                                                                 RGXFWIF_DM_GP,
4410                                                                 psDevInfo->psDeviceNode->psSyncPrim,
4411                                                                 PDUMP_FLAGS_CONTINUOUS);
4412                 PVR_LOGG_IF_ERROR(eError, "RGXWaitForFWOp", return_);
4413
4414                 /* read the OS state */
4415                 OSMemoryBarrier();
4416                 ui32OSStateFlags = *pui32OSStateFlags;
4417
4418                 if ((ui32OSStateFlags & RGXFW_OS_STATE_ACTIVE_OS) == 0)
4419                 {
4420                         /* FW finished offloading the OSID */
4421                         eError = PVRSRV_OK;
4422                         break;
4423                 }
4424                 else
4425                 {
4426                         eError = PVRSRV_ERROR_TIMEOUT;
4427                 }
4428
4429                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4430
4431         } END_LOOP_UNTIL_TIMEOUT();
4432
4433 return_ :
4434 #endif
4435         return eError;
4436 }
4437
4438 PVRSRV_ERROR RGXFWChangeOSidPriority(PVRSRV_RGXDEV_INFO *psDevInfo,
4439                                                                 IMG_UINT32 ui32OSid,
4440                                                                 IMG_UINT32 ui32Priority)
4441 {
4442         PVRSRV_ERROR eError;
4443         RGXFWIF_KCCB_CMD        sOSidPriorityCmd;
4444
4445         sOSidPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_OSID_PRIORITY_CHANGE;
4446         sOSidPriorityCmd.uCmdData.sCmdOSidPriorityData.ui32OSidNum = ui32OSid;
4447         sOSidPriorityCmd.uCmdData.sCmdOSidPriorityData.ui32Priority = ui32Priority;
4448
4449         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4450         {
4451                 eError = RGXScheduleCommand(psDevInfo,
4452                                                                         RGXFWIF_DM_GP,
4453                                                                         &sOSidPriorityCmd,
4454                                                                         sizeof(sOSidPriorityCmd),
4455                                                                         0,
4456                                                                         PDUMP_FLAGS_CONTINUOUS);
4457                 if (eError != PVRSRV_ERROR_RETRY)
4458                 {
4459                         break;
4460                 }
4461                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4462         } END_LOOP_UNTIL_TIMEOUT();
4463
4464         return eError;
4465 }
4466
4467 PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
4468                                                                 CONNECTION_DATA *psConnection,
4469                                                                 PVRSRV_RGXDEV_INFO *psDevInfo,
4470                                                                 IMG_UINT32 ui32Priority,
4471                                                                 RGXFWIF_DM eDM)
4472 {
4473         IMG_UINT32                              ui32CmdSize;
4474         IMG_UINT8                               *pui8CmdPtr;
4475         RGXFWIF_KCCB_CMD                sPriorityCmd;
4476         RGXFWIF_CCB_CMD_HEADER  *psCmdHeader;
4477         RGXFWIF_CMD_PRIORITY    *psCmd;
4478         PVRSRV_ERROR                    eError;
4479
4480         /*
4481                 Get space for command
4482         */
4483         ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_CMD_PRIORITY));
4484
4485         eError = RGXAcquireCCB(FWCommonContextGetClientCCB(psContext),
4486                                                    ui32CmdSize,
4487                                                    (void **) &pui8CmdPtr,
4488                                                    PDUMP_FLAGS_CONTINUOUS);
4489         if (eError != PVRSRV_OK)
4490         {
4491                 if(eError != PVRSRV_ERROR_RETRY)
4492                 {
4493                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire space for client CCB", __FUNCTION__));
4494                 }
4495                 goto fail_ccbacquire;
4496         }
4497
4498         /*
4499                 Write the command header and command
4500         */
4501         psCmdHeader = (RGXFWIF_CCB_CMD_HEADER *) pui8CmdPtr;
4502         psCmdHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PRIORITY;
4503         psCmdHeader->ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CMD_PRIORITY));
4504         pui8CmdPtr += sizeof(*psCmdHeader);
4505
4506         psCmd = (RGXFWIF_CMD_PRIORITY *) pui8CmdPtr;
4507         psCmd->ui32Priority = ui32Priority;
4508         pui8CmdPtr += sizeof(*psCmd);
4509
4510         /*
4511                 We should reserved space in the kernel CCB here and fill in the command
4512                 directly.
4513                 This is so if there isn't space in the kernel CCB we can return with
4514                 retry back to services client before we take any operations
4515         */
4516
4517         /*
4518                 Submit the command
4519         */
4520         RGXReleaseCCB(FWCommonContextGetClientCCB(psContext),
4521                                   ui32CmdSize,
4522                                   PDUMP_FLAGS_CONTINUOUS);
4523
4524         /* Construct the priority command. */
4525         sPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
4526         sPriorityCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psContext);
4527         sPriorityCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psContext));
4528         sPriorityCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
4529         sPriorityCmd.uCmdData.sCmdKickData.sWorkloadDataFWAddress.ui32Addr = 0;
4530         sPriorityCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
4531
4532         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
4533         {
4534                 eError = RGXScheduleCommand(psDevInfo,
4535                                                                         eDM,
4536                                                                         &sPriorityCmd,
4537                                                                         sizeof(sPriorityCmd),
4538                                                                         0,
4539                                                                         PDUMP_FLAGS_CONTINUOUS);
4540                 if (eError != PVRSRV_ERROR_RETRY)
4541                 {
4542                         break;
4543                 }
4544                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
4545         } END_LOOP_UNTIL_TIMEOUT();
4546
4547         if (eError != PVRSRV_OK)
4548         {
4549                 PVR_DPF((PVR_DBG_ERROR,"ContextSetPriority: Failed to submit set priority command with error (%u)", eError));
4550         }
4551
4552         return PVRSRV_OK;
4553
4554 fail_ccbacquire:
4555         PVR_ASSERT(eError != PVRSRV_OK);
4556         return eError;
4557 }
4558
4559 /*
4560         RGXReadMETAAddr
4561 */
4562 PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
4563 {
4564         IMG_UINT8 *pui8RegBase = (IMG_UINT8*)psDevInfo->pvRegsBaseKM;
4565         IMG_UINT32 ui32Value;
4566
4567         /* Wait for Slave Port to be Ready */
4568         if (PVRSRVPollForValueKM(
4569                 (IMG_UINT32*) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
4570                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
4571                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN) != PVRSRV_OK)
4572         {
4573                 return PVRSRV_ERROR_TIMEOUT;
4574         }
4575
4576         /* Issue the Read */
4577         OSWriteHWReg32(
4578             psDevInfo->pvRegsBaseKM,
4579             RGX_CR_META_SP_MSLVCTRL0,
4580             ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
4581
4582         /* Wait for Slave Port to be Ready: read complete */
4583         if (PVRSRVPollForValueKM(
4584                 (IMG_UINT32*) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
4585                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
4586                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN) != PVRSRV_OK)
4587         {
4588                 return PVRSRV_ERROR_TIMEOUT;
4589         }
4590
4591         /* Read the value */
4592         ui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
4593
4594         *pui32Value = ui32Value;
4595
4596         return PVRSRV_OK;
4597 }
4598
4599
4600 /*
4601         RGXUpdateHealthStatus
4602 */
4603 PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
4604                                    IMG_BOOL bCheckAfterTimePassed)
4605 {
4606 #if !defined(PVRSRV_GPUVIRT_GUESTDRV)
4607         PVRSRV_DATA*                 psPVRSRVData = PVRSRVGetPVRSRVData();
4608         PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
4609         PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
4610         PVRSRV_RGXDEV_INFO*  psDevInfo;
4611         RGXFWIF_TRACEBUF*  psRGXFWIfTraceBufCtl;
4612         RGXFWIF_CCB_CTL *psKCCBCtl;
4613         IMG_UINT32  ui32ThreadCount;
4614         IMG_BOOL  bKCCBCmdsWaiting;
4615
4616         PVR_ASSERT(psDevNode != NULL);
4617         psDevInfo = psDevNode->pvDevice;
4618         psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
4619
4620         /* If the firmware is not initialised, there is not much point continuing! */
4621         if (!psDevInfo->bFirmwareInitialised  ||  psDevInfo->pvRegsBaseKM == NULL  ||
4622             psDevInfo->psDeviceNode == NULL)
4623         {
4624                 return PVRSRV_OK;
4625         }
4626
4627         /* If Rogue is not powered on, don't continue
4628            (there is a race condition where PVRSRVIsDevicePowered returns TRUE when the GPU is actually powering down.
4629            That's not a problem as this function does not touch the HW except for the RGXScheduleCommand function,
4630            which is already powerlock safe. The worst thing that could happen is that Rogue might power back up
4631            but the chances of that are very low */
4632         if (!PVRSRVIsDevicePowered(psDevNode))
4633         {
4634                 return PVRSRV_OK;
4635         }
4636
4637         /* If this is a quick update, then include the last current value... */
4638         if (!bCheckAfterTimePassed)
4639         {
4640                 eNewStatus = OSAtomicRead(&psDevNode->eHealthStatus);
4641                 eNewReason = OSAtomicRead(&psDevNode->eHealthReason);
4642         }
4643
4644         /*
4645            Firmware thread checks...
4646         */
4647         for (ui32ThreadCount = 0;  ui32ThreadCount < RGXFW_THREAD_NUM;  ui32ThreadCount++)
4648         {
4649                 if (psRGXFWIfTraceBufCtl != NULL)
4650                 {
4651                         IMG_CHAR*  pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
4652
4653                         /*
4654                         Check if the FW has hit an assert...
4655                         */
4656                         if (*pszTraceAssertInfo != '\0')
4657                         {
4658                                 PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Firmware thread %d has asserted: %s (%s:%d)",
4659                                         ui32ThreadCount, pszTraceAssertInfo,
4660                                                 psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
4661                                                 psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
4662                                 eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
4663                                 eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
4664                                 goto _RGXUpdateHealthStatus_Exit;
4665                         }
4666
4667                         /*
4668                            Check the threads to see if they are in the same poll locations as last time...
4669                         */
4670                         if (bCheckAfterTimePassed)
4671                         {
4672                                 if (psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] != 0  &&
4673                                         psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] == psDevInfo->aui32CrLastPollAddr[ui32ThreadCount])
4674                                 {
4675                                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
4676                                                         ui32ThreadCount,
4677                                                         ((psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
4678                                                         psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
4679                                                         psRGXFWIfTraceBufCtl->aui32CrPollMask[ui32ThreadCount]));
4680                                         eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
4681                                         eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
4682                                         goto _RGXUpdateHealthStatus_Exit;
4683                                 }
4684                                 psDevInfo->aui32CrLastPollAddr[ui32ThreadCount] = psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount];
4685                         }
4686                 }
4687         }
4688
4689         /*
4690            Event Object Timeouts check...
4691         */
4692         if (!bCheckAfterTimePassed)
4693         {
4694                 if (psDevInfo->ui32GEOTimeoutsLastTime > 1  &&  psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
4695                 {
4696                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Global Event Object Timeouts have risen (from %d to %d)",
4697                                         psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
4698                         eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
4699                         eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
4700                 }
4701                 psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
4702         }
4703
4704         /*
4705            Check the Kernel CCB pointer is valid. If any commands were waiting last time, then check
4706            that some have executed since then.
4707         */
4708         bKCCBCmdsWaiting = IMG_FALSE;
4709         psKCCBCtl = psDevInfo->psKernelCCBCtl;
4710
4711         if (psKCCBCtl != NULL)
4712         {
4713                 if (psKCCBCtl->ui32ReadOffset > psKCCBCtl->ui32WrapMask  ||
4714                         psKCCBCtl->ui32WriteOffset > psKCCBCtl->ui32WrapMask)
4715                 {
4716                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: KCCB has invalid offset (ROFF=%d WOFF=%d)",
4717                                 psKCCBCtl->ui32ReadOffset, psKCCBCtl->ui32WriteOffset));
4718                         eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
4719                         eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
4720                 }
4721
4722                 if (psKCCBCtl->ui32ReadOffset != psKCCBCtl->ui32WriteOffset)
4723                 {
4724                         bKCCBCmdsWaiting = IMG_TRUE;
4725                 }
4726         }
4727
4728         if (bCheckAfterTimePassed && psDevInfo->psRGXFWIfTraceBuf != NULL)
4729         {
4730                 IMG_UINT32  ui32KCCBCmdsExecuted = psDevInfo->psRGXFWIfTraceBuf->ui32KCCBCmdsExecuted;
4731
4732                 if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
4733                 {
4734                         /*
4735                            If something was waiting last time then the Firmware has stopped processing commands.
4736                         */
4737                         if (psDevInfo->bKCCBCmdsWaitingLastTime)
4738                         {
4739                                 PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: No KCCB commands executed since check!"));
4740                                 eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
4741                                 eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
4742                         }
4743
4744                         /*
4745                            If no commands are currently pending and nothing happened since the last poll, then
4746                            schedule a dummy command to ping the firmware so we know it is alive and processing.
4747                         */
4748                         if (!bKCCBCmdsWaiting)
4749                         {
4750                                 RGXFWIF_KCCB_CMD  sCmpKCCBCmd;
4751                                 PVRSRV_ERROR      eError;
4752
4753                                 sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
4754
4755                                 eError = RGXScheduleCommand(psDevNode->pvDevice,
4756                                                                                         RGXFWIF_DM_GP,
4757                                                                                         &sCmpKCCBCmd,
4758                                                                                         sizeof(sCmpKCCBCmd),
4759                                                                                         0,
4760                                                                                         IMG_TRUE);
4761                                 if (eError != PVRSRV_OK)
4762                                 {
4763                                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Cannot schedule Health Check command! (0x%x)", eError));
4764                                 }
4765                                 else
4766                                 {
4767                                         bKCCBCmdsWaiting = IMG_TRUE;
4768                                 }
4769                         }
4770                 }
4771
4772                 psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
4773                 psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
4774         }
4775
4776         if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
4777         {
4778                 /* Attempt to detect and deal with any stalled client contexts.
4779                  * Currently, ui32StalledClientMask is not a reliable method of detecting a stalled
4780                  * application as the app could just be busy with a long running task,
4781                  * or a lots of smaller workloads. Also the definition of stalled is
4782                  * effectively subject to the timer frequency calling this function
4783                  * (which is a platform config value with no guarantee it is correctly tuned).
4784                  */
4785
4786                 IMG_UINT32 ui32StalledClientMask = 0;
4787
4788                 ui32StalledClientMask |= CheckForStalledClientTransferCtxt(psDevInfo);
4789
4790                 ui32StalledClientMask |= CheckForStalledClientRenderCtxt(psDevInfo);
4791
4792 #if     !defined(UNDER_WDDM)
4793                 if(psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_COMPUTE_BIT_MASK)
4794                 {
4795                         ui32StalledClientMask |= CheckForStalledClientComputeCtxt(psDevInfo);
4796                 }
4797 #endif
4798
4799                 if (psDevInfo->sDevFeatureCfg.ui64Features & RGX_FEATURE_RAY_TRACING_BIT_MASK)
4800                 {
4801                         ui32StalledClientMask |= CheckForStalledClientRayCtxt(psDevInfo);
4802                 }
4803
4804                 /* If at least one DM stalled bit is different than before */
4805                 if (psDevInfo->ui32StalledClientMask ^ ui32StalledClientMask)
4806                 {
4807                         /* Print all the stalled DMs */
4808                         PVR_LOG(("RGXGetDeviceHealthStatus: Possible stalled client contexts detected: %s%s%s%s%s%s%s%s%s",
4809                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_GP),
4810                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TDM_2D),
4811                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TA),
4812                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_3D),
4813                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_CDM),
4814                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_RTU),
4815                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_SHG),
4816                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ2D),
4817                                  RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ3D)));
4818                 }
4819                 psDevInfo->ui32StalledClientMask = ui32StalledClientMask;
4820         }
4821
4822         /*
4823            Finished, save the new status...
4824         */
4825 _RGXUpdateHealthStatus_Exit:
4826         OSAtomicWrite(&psDevNode->eHealthStatus, eNewStatus);
4827         OSAtomicWrite(&psDevNode->eHealthReason, eNewReason);
4828
4829         /*
4830          * Attempt to service the HWPerf buffer to regularly transport idle/periodic
4831          * packets to host buffer.
4832          */
4833         if (psDevNode->pfnServiceHWPerf != NULL)
4834         {
4835                 PVRSRV_ERROR eError = psDevNode->pfnServiceHWPerf(psDevNode);
4836                 if (eError != PVRSRV_OK)
4837                 {
4838                         PVR_DPF((PVR_DBG_WARNING, "DevicesWatchdogThread: "
4839                                          "Error occurred when servicing HWPerf buffer (%d)",
4840                                          eError));
4841                 }
4842         }
4843
4844 #endif
4845         return PVRSRV_OK;
4846 } /* RGXUpdateHealthStatus */
4847
4848 PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM)
4849 {
4850         RGX_CLIENT_CCB  *psCurrentClientCCB = psCurrentServerCommonContext->psClientCCB;
4851
4852         return CheckForStalledCCB(psCurrentClientCCB, eKickTypeDM);
4853 }
4854
4855 void DumpStalledFWCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
4856                                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
4857                                         void *pvDumpDebugFile)
4858 {
4859         RGX_CLIENT_CCB  *psCurrentClientCCB = psCurrentServerCommonContext->psClientCCB;
4860         PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext = psCurrentServerCommonContext->sFWCommonContextFWAddr;
4861
4862 #if defined(PVRSRV_ENABLE_FULL_SYNC_TRACKING) || defined(PVRSRV_ENABLE_FULL_CCB_DUMP)
4863         DumpCCB(psCurrentServerCommonContext->psDevInfo, sFWCommonContext,
4864                         psCurrentClientCCB, pfnDumpDebugPrintf, pvDumpDebugFile);
4865 #else
4866         DumpStalledCCBCommand(sFWCommonContext, psCurrentClientCCB, pfnDumpDebugPrintf, pvDumpDebugFile);
4867 #endif
4868 }
4869
4870 void AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
4871                                                                         IMG_UINT32 *pui32NumCleanupCtl,
4872                                                                         RGXFWIF_DM eDM,
4873                                                                         IMG_BOOL bKick,
4874                                                                         RGX_RTDATA_CLEANUP_DATA        *psRTDataCleanup,
4875                                                                         RGX_ZSBUFFER_DATA              *psZBuffer,
4876                                                                         RGX_ZSBUFFER_DATA              *psSBuffer)
4877 {
4878         PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
4879
4880         PVR_ASSERT((eDM == RGXFWIF_DM_TA) || (eDM == RGXFWIF_DM_3D));
4881
4882         if(bKick)
4883         {
4884                 if(eDM == RGXFWIF_DM_TA)
4885                 {
4886                         if(psRTDataCleanup)
4887                         {
4888                                 PRGXFWIF_CLEANUP_CTL psCleanupCtl;
4889
4890                                 RGXSetFirmwareAddress(&psCleanupCtl, psRTDataCleanup->psFWHWRTDataMemDesc,
4891                                                                         offsetof(RGXFWIF_HWRTDATA, sTACleanupState),
4892                                                                 RFW_FWADDR_NOREF_FLAG);
4893
4894                                 *(psCleanupCtlWrite++) = psCleanupCtl;
4895                         }
4896                 }
4897                 else
4898                 {
4899                         if(psRTDataCleanup)
4900                         {
4901                                 PRGXFWIF_CLEANUP_CTL psCleanupCtl;
4902
4903                                 RGXSetFirmwareAddress(&psCleanupCtl, psRTDataCleanup->psFWHWRTDataMemDesc,
4904                                                                         offsetof(RGXFWIF_HWRTDATA, s3DCleanupState),
4905                                                                 RFW_FWADDR_NOREF_FLAG);
4906
4907                                 *(psCleanupCtlWrite++) = psCleanupCtl;
4908                         }
4909
4910                         if(psZBuffer)
4911                         {
4912                                 (psCleanupCtlWrite++)->ui32Addr = psZBuffer->sZSBufferFWDevVAddr.ui32Addr +
4913                                                                 offsetof(RGXFWIF_FWZSBUFFER, sCleanupState);
4914                         }
4915
4916                         if(psSBuffer)
4917                         {
4918                                 (psCleanupCtlWrite++)->ui32Addr = psSBuffer->sZSBufferFWDevVAddr.ui32Addr +
4919                                                                 offsetof(RGXFWIF_FWZSBUFFER, sCleanupState);
4920                         }
4921                 }
4922         }
4923
4924         *pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
4925
4926         PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
4927 }
4928
4929 PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
4930 {
4931 #if defined(PVRSRV_GPUVIRT_GUESTDRV)
4932         /* Guest drivers do not support HW reset */
4933         PVR_UNREFERENCED_PARAMETER(psDevNode);
4934 #else
4935         PVRSRV_RGXDEV_INFO      *psDevInfo;
4936         RGXFWIF_HWRINFOBUF      *psHWRInfoBuf;
4937         RGXFWIF_TRACEBUF        *psRGXFWIfTraceBufCtl;
4938         IMG_UINT32                      i;
4939
4940         if(psDevNode->pvDevice == NULL)
4941         {
4942                 return PVRSRV_ERROR_INVALID_DEVINFO;
4943         }
4944         psDevInfo = psDevNode->pvDevice;
4945
4946         psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBuf;
4947         psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
4948
4949         for(i = 0 ; i < psDevInfo->sDevFeatureCfg.ui32MAXDMCount ; i++)
4950         {
4951                 /* Reset the HWR numbers */
4952                 psRGXFWIfTraceBufCtl->aui32HwrDmLockedUpCount[i] = 0;
4953                 psRGXFWIfTraceBufCtl->aui32HwrDmFalseDetectCount[i] = 0;
4954                 psRGXFWIfTraceBufCtl->aui32HwrDmRecoveredCount[i] = 0;
4955                 psRGXFWIfTraceBufCtl->aui32HwrDmOverranCount[i] = 0;
4956         }
4957
4958         for(i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
4959         {
4960                 psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
4961         }
4962
4963         for(i = 0 ; i < RGXFW_THREAD_NUM ; i++)
4964         {
4965                 psHWRInfoBuf->ui32FirstCrPollAddr[i] = 0;
4966                 psHWRInfoBuf->ui32FirstCrPollMask[i] = 0;
4967         }
4968
4969         psHWRInfoBuf->ui32WriteIndex = 0;
4970         psHWRInfoBuf->ui32DDReqCount = 0;
4971 #endif
4972         return PVRSRV_OK;
4973 }
4974
4975 PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
4976                                                    IMG_DEV_PHYADDR *psPhyAddr,
4977                                                    IMG_UINT32 ui32LogicalOffset,
4978                                                    IMG_UINT32 ui32Log2PageSize,
4979                                                    IMG_UINT32 ui32NumOfPages,
4980                                                    IMG_BOOL *bValid)
4981 {
4982
4983         PVRSRV_ERROR eError;
4984
4985         eError = PMRLockSysPhysAddresses(psPMR);
4986         if (eError != PVRSRV_OK)
4987         {
4988                 PVR_DPF((PVR_DBG_ERROR,"RGXGetPhyAddr: PMRLockSysPhysAddresses failed (%u)",
4989                                 eError));
4990                 return eError;
4991         }
4992
4993         eError = PMR_DevPhysAddr(psPMR,
4994                                                                  ui32Log2PageSize,
4995                                                                  ui32NumOfPages,
4996                                                                  ui32LogicalOffset,
4997                                                                  psPhyAddr,
4998                                                                  bValid);
4999
5000         if (eError != PVRSRV_OK)
5001         {
5002                 PVR_DPF((PVR_DBG_ERROR,"RGXGetPhyAddr: PMR_DevPhysAddr failed (%u)",
5003                                 eError));
5004                 return eError;
5005         }
5006
5007
5008         eError = PMRUnlockSysPhysAddresses(psPMR);
5009         if (eError != PVRSRV_OK)
5010         {
5011                 PVR_DPF((PVR_DBG_ERROR,"RGXGetPhyAddr: PMRUnLockSysPhysAddresses failed (%u)",
5012                                 eError));
5013                 return eError;
5014         }
5015
5016         return eError;
5017 }
5018
5019 #if defined(PDUMP)
5020 PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset)
5021 {
5022         RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
5023         PVRSRV_ERROR eError = PVRSRV_OK;
5024
5025         if (psDevInfo->bDumpedKCCBCtlAlready)
5026         {
5027                 /* exiting capture range */
5028                 psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
5029
5030                 /* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
5031                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
5032                                       "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
5033                                       psKCCBCtl,
5034                                       ui32WriteOffset,
5035                                       ui32WriteOffset);
5036                 eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
5037                                                 offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
5038                                                 ui32WriteOffset,
5039                                                 0xffffffff,
5040                                                 PDUMP_POLL_OPERATOR_EQUAL,
5041                                                 PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
5042
5043                 if (eError != PVRSRV_OK)
5044                 {
5045                         PVR_DPF((PVR_DBG_ERROR, "RGXPdumpDrainKCCB: problem pdumping POL for kCCBCtl (%d)", eError));
5046                 }
5047         }
5048
5049         return eError;
5050
5051 }
5052 #endif
5053
5054 /*!
5055 *******************************************************************************
5056
5057  @Function      RGXClientConnectCompatCheck_ClientAgainstFW
5058
5059  @Description
5060
5061  Check compatibility of client and firmware (build options)
5062  at the connection time.
5063
5064  @Input psDeviceNode - device node
5065  @Input ui32ClientBuildOptions - build options for the client
5066
5067  @Return   PVRSRV_ERROR - depending on mismatch found
5068
5069 ******************************************************************************/
5070 PVRSRV_ERROR IMG_CALLCONV RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE * psDeviceNode, IMG_UINT32 ui32ClientBuildOptions)
5071 {
5072         PVRSRV_ERROR            eError;
5073 #if defined(PVRSRV_GPUVIRT_GUESTDRV)
5074         eError = PVRSRV_OK;
5075 #else
5076 #if !defined(NO_HARDWARE) || defined(PDUMP)
5077         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
5078 #endif
5079 #if !defined(NO_HARDWARE)
5080         RGXFWIF_INIT    *psRGXFWInit = NULL;
5081         IMG_UINT32              ui32BuildOptionsMismatch;
5082         IMG_UINT32              ui32BuildOptionsFW;
5083
5084         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc,
5085                                                                                                 (void **)&psRGXFWInit);
5086         if (eError != PVRSRV_OK)
5087         {
5088                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to acquire kernel fw compatibility check info (%u)",
5089                                 __FUNCTION__, eError));
5090                 return eError;
5091         }
5092
5093         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
5094         {
5095                 if(*((volatile IMG_BOOL *)&psRGXFWInit->sRGXCompChecks.bUpdated))
5096                 {
5097                         /* No need to wait if the FW has already updated the values */
5098                         break;
5099                 }
5100                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
5101         } END_LOOP_UNTIL_TIMEOUT();
5102 #endif
5103
5104 #if defined(PDUMP)
5105         PDUMPCOMMENT("Compatibility check: client and FW build options");
5106         eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfInitMemDesc,
5107                                                                                                 offsetof(RGXFWIF_INIT, sRGXCompChecks) +
5108                                                                                                 offsetof(RGXFWIF_COMPCHECKS, ui32BuildOptions),
5109                                                                                                 ui32ClientBuildOptions,
5110                                                                                                 0xffffffff,
5111                                                                                                 PDUMP_POLL_OPERATOR_EQUAL,
5112                                                                                                 PDUMP_FLAGS_CONTINUOUS);
5113         if (eError != PVRSRV_OK)
5114         {
5115                 PVR_DPF((PVR_DBG_ERROR, "RGXDevInitCompatCheck: problem pdumping POL for psRGXFWIfInitMemDesc (%d)", eError));
5116                 return eError;
5117         }
5118 #endif
5119
5120 #if !defined(NO_HARDWARE)
5121         if (psRGXFWInit == NULL)
5122         {
5123                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to acquire kernel fw compatibility check info, psRGXFWInit is NULL", __FUNCTION__));
5124                 eError = PVRSRV_ERROR_INVALID_PARAMS;
5125                 goto chk_exit;
5126         }
5127
5128         ui32BuildOptionsFW = psRGXFWInit->sRGXCompChecks.ui32BuildOptions;
5129         ui32BuildOptionsMismatch = ui32ClientBuildOptions ^ ui32BuildOptionsFW;
5130
5131         if (ui32BuildOptionsMismatch != 0)
5132         {
5133                 if ( (ui32ClientBuildOptions & ui32BuildOptionsMismatch) != 0)
5134                 {
5135                         PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
5136                         "extra options present in client: (0x%x). Please check rgx_options.h",
5137                         ui32ClientBuildOptions & ui32BuildOptionsMismatch ));
5138                 }
5139
5140                 if ( (ui32BuildOptionsFW & ui32BuildOptionsMismatch) != 0)
5141                 {
5142                         PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
5143                         "extra options present in Firmware: (0x%x). Please check rgx_options.h",
5144                         ui32BuildOptionsFW & ui32BuildOptionsMismatch ));
5145                 }
5146                 eError = PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
5147                 goto chk_exit;
5148         }
5149         else
5150         {
5151                 PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: Firmware and client build options match. [ OK ]"));
5152         }
5153 #endif
5154
5155         eError = PVRSRV_OK;
5156 #if !defined(NO_HARDWARE)
5157 chk_exit:
5158         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
5159 #endif
5160 #endif
5161         return eError;
5162
5163 }
5164
5165 /******************************************************************************
5166  End of file (rgxfwutils.c)
5167 ******************************************************************************/