drivers/staging/imgtec/rogue/physmem_lma.c

   1 /*************************************************************************/ /*!
   2 @File           physmem_lma.c
   3 @Title          Local card memory allocator
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Part of the memory management. This module is responsible for
   6                 implementing the function callbacks for local card memory.
   7 @License        Dual MIT/GPLv2
   8
   9 The contents of this file are subject to the MIT license as set out below.
  10
  11 Permission is hereby granted, free of charge, to any person obtaining a copy
  12 of this software and associated documentation files (the "Software"), to deal
  13 in the Software without restriction, including without limitation the rights
  14 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  15 copies of the Software, and to permit persons to whom the Software is
  16 furnished to do so, subject to the following conditions:
  17
  18 The above copyright notice and this permission notice shall be included in
  19 all copies or substantial portions of the Software.
  20
  21 Alternatively, the contents of this file may be used under the terms of
  22 the GNU General Public License Version 2 ("GPL") in which case the provisions
  23 of GPL are applicable instead of those above.
  24
  25 If you wish to allow use of your version of this file only under the terms of
  26 GPL, and not to allow others to use your version of this file under the terms
  27 of the MIT license, indicate your decision by deleting the provisions above
  28 and replace them with the notice and other provisions required by GPL as set
  29 out in the file called "GPL-COPYING" included in this distribution. If you do
  30 not delete the provisions above, a recipient may use your version of this file
  31 under the terms of either the MIT license or GPL.
  32
  33 This License is also included in this distribution in the file called
  34 "MIT-COPYING".
  35
  36 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  37 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  38 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  39 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  40 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  41 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  42 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  43 */ /**************************************************************************/
  44
  45 #include "img_types.h"
  46 #include "pvr_debug.h"
  47 #include "pvrsrv_error.h"
  48 #include "pvrsrv_memallocflags.h"
  49 #include "rgx_pdump_panics.h"
  50 #include "allocmem.h"
  51 #include "osfunc.h"
  52 #include "pvrsrv.h"
  53 #include "devicemem_server_utils.h"
  54 #include "physmem_lma.h"
  55 #include "pdump_km.h"
  56 #include "pmr.h"
  57 #include "pmr_impl.h"
  58 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
  59 #include "process_stats.h"
  60 #endif
  61
  62 #if defined(SUPPORT_GPUVIRT_VALIDATION)
  63 #include "rgxutils.h"
  64 #endif
  65
  66 typedef struct _PMR_LMALLOCARRAY_DATA_ {
  67         PVRSRV_DEVICE_NODE *psDevNode;
  68     IMG_INT32 iNumPagesAllocated;
  69     /*
  70      * uiTotalNumPages:
  71      * Total number of pages supported by this PMR. (Fixed as of now due the fixed Page table array size)
  72      */
  73     IMG_UINT32 uiTotalNumPages;
  74     IMG_UINT32 uiPagesToAlloc;
  75
  76         IMG_UINT32 uiLog2AllocSize;
  77         IMG_UINT32 uiAllocSize;
  78         IMG_DEV_PHYADDR *pasDevPAddr;
  79
  80         IMG_BOOL bZeroOnAlloc;
  81         IMG_BOOL bPoisonOnAlloc;
  82         IMG_BOOL bFwLocalAlloc;
  83
  84         /* Tells if allocation is physically backed */
  85         IMG_BOOL bHasLMPages;
  86         IMG_BOOL bOnDemand;
  87
  88         /*
  89           record at alloc time whether poisoning will be required when the
  90           PMR is freed.
  91         */
  92         IMG_BOOL bPoisonOnFree;
  93
  94         /* Physical heap and arena pointers for this allocation */
  95         PHYS_HEAP* psPhysHeap;
  96         RA_ARENA* psArena;
  97         PVRSRV_MEMALLOCFLAGS_T uiAllocFlags;
  98
  99 } PMR_LMALLOCARRAY_DATA;
 100
 101 static PVRSRV_ERROR _MapAlloc(PVRSRV_DEVICE_NODE *psDevNode,
 102                                                           IMG_DEV_PHYADDR *psDevPAddr,
 103                                                           size_t uiSize,
 104                                                           IMG_BOOL bFwLocalAlloc,
 105                                                           PMR_FLAGS_T ulFlags,
 106                                                           void **pvPtr)
 107 {
 108         IMG_UINT32 ui32CPUCacheFlags = DevmemCPUCacheMode(psDevNode, ulFlags);
 109         IMG_CPU_PHYADDR sCpuPAddr;
 110         PHYS_HEAP *psPhysHeap;
 111
 112         if (bFwLocalAlloc)
 113         {
 114                 psPhysHeap = psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_FW_LOCAL];
 115         }
 116         else
 117         {
 118                 psPhysHeap = psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_GPU_LOCAL];
 119         }
 120
 121         PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPAddr, psDevPAddr);
 122
 123         *pvPtr = OSMapPhysToLin(sCpuPAddr, uiSize, ui32CPUCacheFlags);
 124         if (*pvPtr == NULL)
 125         {
 126                 return PVRSRV_ERROR_OUT_OF_MEMORY;
 127         }
 128         else
 129         {
 130                 return PVRSRV_OK;
 131         }
 132 }
 133
 134 static void _UnMapAlloc(PVRSRV_DEVICE_NODE *psDevNode,
 135                                                 size_t uiSize,
 136                                                 IMG_BOOL bFwLocalAlloc,
 137                                                 PMR_FLAGS_T ulFlags,
 138                                                 void *pvPtr)
 139 {
 140         OSUnMapPhysToLin(pvPtr, uiSize, PVRSRV_CPU_CACHE_MODE(ulFlags));
 141 }
 142
 143 static PVRSRV_ERROR
 144 _PoisonAlloc(PVRSRV_DEVICE_NODE *psDevNode,
 145                          IMG_DEV_PHYADDR *psDevPAddr,
 146                          IMG_BOOL bFwLocalAlloc,
 147                          IMG_UINT32 uiAllocSize,
 148                          const IMG_CHAR *pacPoisonData,
 149                          size_t uiPoisonSize)
 150 {
 151         IMG_UINT32 uiSrcByteIndex;
 152         IMG_UINT32 uiDestByteIndex;
 153         void *pvKernLin = NULL;
 154         IMG_CHAR *pcDest = NULL;
 155
 156         PVRSRV_ERROR eError;
 157
 158         eError = _MapAlloc(psDevNode,
 159                                            psDevPAddr,
 160                                            uiAllocSize,
 161                                            bFwLocalAlloc,
 162                                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
 163                                            &pvKernLin);
 164         if (eError != PVRSRV_OK)
 165         {
 166                 goto map_failed;
 167         }
 168         pcDest = pvKernLin;
 169
 170         uiSrcByteIndex = 0;
 171         for(uiDestByteIndex=0; uiDestByteIndex<uiAllocSize; uiDestByteIndex++)
 172         {
 173                 pcDest[uiDestByteIndex] = pacPoisonData[uiSrcByteIndex];
 174                 uiSrcByteIndex++;
 175                 if (uiSrcByteIndex == uiPoisonSize)
 176                 {
 177                         uiSrcByteIndex = 0;
 178                 }
 179         }
 180
 181         _UnMapAlloc(psDevNode, uiAllocSize, bFwLocalAlloc, 0,pvKernLin);
 182
 183         return PVRSRV_OK;
 184
 185 map_failed:
 186         PVR_DPF((PVR_DBG_ERROR, "Failed to poison allocation"));
 187         return eError;
 188 }
 189
 190 static PVRSRV_ERROR
 191 _ZeroAlloc(PVRSRV_DEVICE_NODE *psDevNode,
 192                    IMG_DEV_PHYADDR *psDevPAddr,
 193                    IMG_BOOL bFwLocalAlloc,
 194                    IMG_UINT32 uiAllocSize)
 195 {
 196         void *pvKernLin = NULL;
 197         PVRSRV_ERROR eError;
 198
 199         eError = _MapAlloc(psDevNode,
 200                                            psDevPAddr,
 201                                            uiAllocSize,
 202                                            bFwLocalAlloc,
 203                                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
 204                                            &pvKernLin);
 205         if (eError != PVRSRV_OK)
 206         {
 207                 goto map_failed;
 208         }
 209
 210         /* NOTE: 'CachedMemSet' means the operating system default memset, which
 211          *       we *assume* in the LMA code will be faster, and doesn't need to
 212          *       worry about ARM64.
 213          */
 214         OSCachedMemSet(pvKernLin, 0, uiAllocSize);
 215
 216         _UnMapAlloc(psDevNode, uiAllocSize, bFwLocalAlloc, 0, pvKernLin);
 217
 218         return PVRSRV_OK;
 219
 220 map_failed:
 221         PVR_DPF((PVR_DBG_ERROR, "Failed to zero allocation"));
 222         return eError;
 223 }
 224
 225 static const IMG_CHAR _AllocPoison[] = "^PoIsOn";
 226 static const IMG_UINT32 _AllocPoisonSize = 7;
 227 static const IMG_CHAR _FreePoison[] = "<DEAD-BEEF>";
 228 static const IMG_UINT32 _FreePoisonSize = 11;
 229
 230 static PVRSRV_ERROR
 231 _AllocLMPageArray(PVRSRV_DEVICE_NODE *psDevNode,
 232                           PMR_SIZE_T uiSize,
 233                           PMR_SIZE_T uiChunkSize,
 234                           IMG_UINT32 ui32NumPhysChunks,
 235                           IMG_UINT32 ui32NumVirtChunks,
 236                           IMG_UINT32 *pabMappingTable,
 237                           IMG_UINT32 uiLog2PageSize,
 238                           IMG_BOOL bZero,
 239                           IMG_BOOL bPoisonOnAlloc,
 240                           IMG_BOOL bPoisonOnFree,
 241                           IMG_BOOL bContig,
 242                           IMG_BOOL bOnDemand,
 243                           IMG_BOOL bFwLocalAlloc,
 244                           PHYS_HEAP* psPhysHeap,
 245                           PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
 246                           PMR_LMALLOCARRAY_DATA **ppsPageArrayDataPtr
 247                           )
 248 {
 249         PMR_LMALLOCARRAY_DATA *psPageArrayData = NULL;
 250         PVRSRV_ERROR eError;
 251
 252         PVR_ASSERT(!bZero || !bPoisonOnAlloc);
 253
 254         if (uiSize >= 0x1000000000ULL)
 255         {
 256                 PVR_DPF((PVR_DBG_ERROR,
 257                                  "physmem_lma.c: Do you really want 64GB of physical memory in one go?  This is likely a bug"));
 258                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 259                 goto errorOnParam;
 260         }
 261
 262         PVR_ASSERT(OSGetPageShift() <= uiLog2PageSize);
 263
 264         if ((uiSize & ((1ULL << uiLog2PageSize) - 1)) != 0)
 265         {
 266                 eError = PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
 267                 goto errorOnParam;
 268         }
 269
 270         psPageArrayData = OSAllocZMem(sizeof(PMR_LMALLOCARRAY_DATA));
 271         if (psPageArrayData == NULL)
 272         {
 273                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 274                 goto errorOnAllocArray;
 275         }
 276
 277         if (bContig)
 278         {
 279                 /*
 280                         Some allocations require kernel mappings in which case in order
 281                         to be virtually contiguous we also have to be physically contiguous.
 282                 */
 283                 psPageArrayData->uiPagesToAlloc = psPageArrayData->uiTotalNumPages = 1;
 284                 psPageArrayData->uiAllocSize = TRUNCATE_64BITS_TO_32BITS(uiSize);
 285                 psPageArrayData->uiLog2AllocSize = uiLog2PageSize;
 286         }
 287         else
 288         {
 289                 IMG_UINT32 uiNumPages;
 290
 291                 /* Use of cast below is justified by the assertion that follows to
 292                 prove that no significant bits have been truncated */
 293                 uiNumPages = (IMG_UINT32)(((uiSize-1)>>uiLog2PageSize) + 1);
 294                 PVR_ASSERT(((PMR_SIZE_T)uiNumPages << uiLog2PageSize) == uiSize);
 295                 psPageArrayData->uiTotalNumPages = uiNumPages;
 296                 if((1 == ui32NumPhysChunks) && (1 == ui32NumVirtChunks))
 297                 {
 298                         psPageArrayData->uiPagesToAlloc = uiNumPages;
 299                 }else{
 300                         psPageArrayData->uiPagesToAlloc = ui32NumPhysChunks;
 301                 }
 302                 psPageArrayData->uiAllocSize = 1 << uiLog2PageSize;
 303                 psPageArrayData->uiLog2AllocSize = uiLog2PageSize;
 304         }
 305         psPageArrayData->psDevNode = psDevNode;
 306         psPageArrayData->pasDevPAddr = OSAllocMem(sizeof(IMG_DEV_PHYADDR)*
 307                                                                                                 psPageArrayData->uiTotalNumPages);
 308         if (psPageArrayData->pasDevPAddr == NULL)
 309         {
 310                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 311                 goto errorOnAllocAddr;
 312         }
 313
 314         OSCachedMemSet(&psPageArrayData->pasDevPAddr[0], INVALID_PAGE, sizeof(IMG_DEV_PHYADDR)*
 315                                                                                                 psPageArrayData->uiTotalNumPages);
 316
 317     psPageArrayData->iNumPagesAllocated = 0;
 318     psPageArrayData->bZeroOnAlloc = bZero;
 319         psPageArrayData->bPoisonOnAlloc = bPoisonOnAlloc;
 320         psPageArrayData->bPoisonOnFree = bPoisonOnFree;
 321         psPageArrayData->bHasLMPages = IMG_FALSE;
 322         psPageArrayData->bOnDemand = bOnDemand;
 323         psPageArrayData->bFwLocalAlloc = bFwLocalAlloc;
 324         psPageArrayData->psPhysHeap = psPhysHeap;
 325         psPageArrayData->uiAllocFlags = uiAllocFlags;
 326
 327         *ppsPageArrayDataPtr = psPageArrayData;
 328
 329         return PVRSRV_OK;
 330
 331         /*
 332           error exit paths follow:
 333         */
 334
 335 errorOnAllocAddr:
 336         OSFreeMem(psPageArrayData);
 337
 338 errorOnAllocArray:
 339 errorOnParam:
 340         PVR_ASSERT(eError != PVRSRV_OK);
 341         return eError;
 342 }
 343
 344
 345 static PVRSRV_ERROR
 346 _AllocLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
 347 {
 348         PVRSRV_ERROR eError;
 349         RA_BASE_T uiCardAddr;
 350         RA_LENGTH_T uiActualSize;
 351         IMG_UINT32 i,ui32Index=0;
 352         IMG_UINT32 uiAllocSize;
 353         IMG_UINT32 uiLog2AllocSize;
 354         IMG_UINT32 uiRegionId;
 355         PVRSRV_DEVICE_NODE *psDevNode;
 356         IMG_BOOL bPoisonOnAlloc;
 357         IMG_BOOL bZeroOnAlloc;
 358         RA_ARENA *pArena;
 359
 360         PVR_ASSERT(NULL != psPageArrayData);
 361         PVR_ASSERT(0 <= psPageArrayData->iNumPagesAllocated);
 362
 363         uiAllocSize = psPageArrayData->uiAllocSize;
 364         uiLog2AllocSize = psPageArrayData->uiLog2AllocSize;
 365         psDevNode = psPageArrayData->psDevNode;
 366         bPoisonOnAlloc =  psPageArrayData->bPoisonOnAlloc;
 367         bZeroOnAlloc =  psPageArrayData->bZeroOnAlloc;
 368
 369 #if defined(SUPPORT_PVRSRV_GPUVIRT)
 370         if (psPageArrayData->bFwLocalAlloc)
 371         {
 372                 PVR_ASSERT(psDevNode->uiKernelFwRAIdx < RGXFW_NUM_OS);
 373                 pArena = psDevNode->psKernelFwMemArena[psDevNode->uiKernelFwRAIdx];
 374                 psDevNode->uiKernelFwRAIdx = 0;
 375         }
 376         else
 377 #endif
 378         {
 379                 /* Get suitable local memory region for this allocation */
 380                 uiRegionId = PhysHeapGetRegionId(psPageArrayData->psPhysHeap, psPageArrayData->uiAllocFlags);
 381
 382                 PVR_ASSERT(uiRegionId < psDevNode->ui32NumOfLocalMemArenas);
 383                 pArena = psDevNode->apsLocalDevMemArenas[uiRegionId];
 384         }
 385
 386         if(psPageArrayData->uiTotalNumPages < (psPageArrayData->iNumPagesAllocated + psPageArrayData->uiPagesToAlloc))
 387         {
 388                 PVR_DPF((PVR_DBG_ERROR,"Pages requested to allocate larger than original PMR alloc Size"));
 389                 eError = PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
 390                 return eError;
 391         }
 392
 393
 394 #if defined(SUPPORT_GPUVIRT_VALIDATION)
 395         {
 396                 IMG_UINT32  ui32OSid=0, ui32OSidReg=0;
 397                 IMG_BOOL    bOSidAxiProt;
 398                 IMG_PID     pId;
 399
 400                 pId=OSGetCurrentClientProcessIDKM();
 401                 RetrieveOSidsfromPidList(pId, &ui32OSid, &ui32OSidReg, &bOSidAxiProt);
 402
 403                 pArena=psDevNode->psOSidSubArena[ui32OSid];
 404                 PVR_DPF((PVR_DBG_MESSAGE,"(GPU Virtualization Validation): Giving from OS slot %d",ui32OSid));
 405         }
 406 #endif
 407
 408         psPageArrayData->psArena = pArena;
 409
 410         for(i=0;i<psPageArrayData->uiPagesToAlloc;i++)
 411         {
 412
 413                 /*This part of index finding should happen before allocating page. Just avoiding intricate paths */
 414                 if(psPageArrayData->uiTotalNumPages == psPageArrayData->uiPagesToAlloc)
 415                 {
 416                         ui32Index = i;
 417                 }
 418                 else
 419                 {
 420                         if(NULL == pui32MapTable)
 421                         {
 422                                 PVR_DPF((PVR_DBG_MESSAGE,"Mapping table cannot be null"));
 423                                 eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
 424                                 goto errorOnRAAlloc;
 425                         }
 426
 427                         ui32Index = pui32MapTable[i];
 428                         if(ui32Index >= psPageArrayData->uiTotalNumPages)
 429                         {
 430                                 PVR_DPF((PVR_DBG_MESSAGE, "%s: Page alloc request Index out of bounds for PMR @0x%p",__func__, psPageArrayData));
 431                                 eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
 432                                 goto errorOnRAAlloc;
 433                         }
 434
 435                         if(INVALID_PAGE != psPageArrayData->pasDevPAddr[ui32Index].uiAddr)
 436                         {
 437                                 PVR_DPF((PVR_DBG_MESSAGE,"Mapping already exists"));
 438                                 eError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
 439                                 goto errorOnRAAlloc;
 440                         }
 441                 }
 442
 443                 eError = RA_Alloc(pArena,
 444                                   uiAllocSize,
 445                                   RA_NO_IMPORT_MULTIPLIER,
 446                                   0,                       /* No flags */
 447                                   1ULL << uiLog2AllocSize,
 448                                   "LMA_Page_Alloc",
 449                                   &uiCardAddr,
 450                                   &uiActualSize,
 451                                   NULL);                   /* No private handle */
 452
 453 #if defined(SUPPORT_GPUVIRT_VALIDATION)
 454 {
 455                 PVR_DPF((PVR_DBG_MESSAGE,"(GPU Virtualization Validation): Address: %llu \n",uiCardAddr));
 456 }
 457 #endif
 458
 459                 if (PVRSRV_OK != eError)
 460                 {
 461                         PVR_DPF((PVR_DBG_ERROR,"Failed to Allocate the page @index:%d",ui32Index));
 462                         eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
 463                         goto errorOnRAAlloc;
 464                 }
 465
 466 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
 467 #if !defined(PVRSRV_ENABLE_MEMORY_STATS)
 468                 /* Allocation is done a page at a time */
 469                 PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, uiActualSize);
 470 #else
 471                 {
 472                         IMG_CPU_PHYADDR sLocalCpuPAddr;
 473
 474                         sLocalCpuPAddr.uiAddr = (IMG_UINT64)uiCardAddr;
 475                         PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
 476                                                                          NULL,
 477                                                                          sLocalCpuPAddr,
 478                                                                          uiActualSize,
 479                                                                          NULL);
 480                 }
 481 #endif
 482 #endif
 483
 484                 psPageArrayData->pasDevPAddr[ui32Index].uiAddr = uiCardAddr;
 485                 if (bPoisonOnAlloc)
 486                 {
 487                         eError = _PoisonAlloc(psDevNode,
 488                                                                   &psPageArrayData->pasDevPAddr[i],
 489                                                                   psPageArrayData->bFwLocalAlloc,
 490                                                                   uiAllocSize,
 491                                                                   _AllocPoison,
 492                                                                   _AllocPoisonSize);
 493                         if (eError !=PVRSRV_OK)
 494                         {
 495                                 PVR_DPF((PVR_DBG_ERROR,"Failed to poison the page"));
 496                                 goto errorOnPoison;
 497                         }
 498                 }
 499
 500                 if (bZeroOnAlloc)
 501                 {
 502                         eError = _ZeroAlloc(psDevNode,
 503                                                                 &psPageArrayData->pasDevPAddr[i],
 504                                                                 psPageArrayData->bFwLocalAlloc,
 505                                                                 uiAllocSize);
 506                         if (eError !=PVRSRV_OK)
 507                         {
 508                                 PVR_DPF((PVR_DBG_ERROR,"Failed to zero the page"));
 509                                 goto errorOnZero;
 510                         }
 511                 }
 512         }
 513         psPageArrayData->iNumPagesAllocated += psPageArrayData->uiPagesToAlloc;
 514         if(psPageArrayData->iNumPagesAllocated)
 515         {
 516                 psPageArrayData->bHasLMPages = IMG_TRUE;
 517         }
 518
 519         return PVRSRV_OK;
 520
 521         /*
 522           error exit paths follow:
 523         */
 524 errorOnZero:
 525 errorOnPoison:
 526         eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
 527 errorOnRAAlloc:
 528 PVR_DPF((PVR_DBG_ERROR,
 529                      "%s: alloc_pages failed to honour request %d @index: %d of %d pages: (%s)",__func__,
 530                      ui32Index,
 531                      i,
 532                      psPageArrayData->uiPagesToAlloc,
 533                      PVRSRVGetErrorStringKM(eError)));
 534         while (--i < psPageArrayData->uiPagesToAlloc)
 535         {
 536                 if(psPageArrayData->uiTotalNumPages == psPageArrayData->uiPagesToAlloc)
 537                 {
 538                         ui32Index = i;
 539                 }
 540                 else
 541                 {
 542                         if(NULL != pui32MapTable)
 543                                 ui32Index = pui32MapTable[i];
 544                 }
 545
 546                 if(ui32Index < psPageArrayData->uiTotalNumPages)
 547                 {
 548                         RA_Free(pArena, psPageArrayData->pasDevPAddr[ui32Index].uiAddr);
 549                         psPageArrayData->pasDevPAddr[ui32Index].uiAddr = INVALID_PAGE;
 550                 }
 551         }
 552         PVR_ASSERT(eError != PVRSRV_OK);
 553         return eError;
 554 }
 555
 556 static PVRSRV_ERROR
 557 _FreeLMPageArray(PMR_LMALLOCARRAY_DATA *psPageArrayData)
 558 {
 559         OSFreeMem(psPageArrayData->pasDevPAddr);
 560
 561         PVR_DPF((PVR_DBG_MESSAGE, "physmem_lma.c: freed local memory array structure for PMR @0x%p", psPageArrayData));
 562
 563         OSFreeMem(psPageArrayData);
 564
 565         return PVRSRV_OK;
 566 }
 567
 568 static PVRSRV_ERROR
 569 _FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,IMG_UINT32 *pui32FreeIndices, IMG_UINT32 ui32FreePageCount)
 570 {
 571         IMG_UINT32 uiAllocSize;
 572         IMG_UINT32 i,ui32PagesToFree=0,ui32PagesFreed=0,ui32Index=0;
 573         RA_ARENA *pArena = psPageArrayData->psArena;
 574
 575 #if defined(SUPPORT_PVRSRV_GPUVIRT)
 576         PVRSRV_DEVICE_NODE *psDevNode = psPageArrayData->psDevNode;
 577         if (psPageArrayData->bFwLocalAlloc)
 578         {
 579                 PVR_ASSERT(psDevNode->uiKernelFwRAIdx < RGXFW_NUM_OS);
 580                 pArena = psDevNode->psKernelFwMemArena[psDevNode->uiKernelFwRAIdx];
 581                 psDevNode->uiKernelFwRAIdx = 0;
 582         }
 583 #endif
 584
 585         PVR_ASSERT(psPageArrayData->bHasLMPages);
 586
 587         uiAllocSize = psPageArrayData->uiAllocSize;
 588
 589         ui32PagesToFree = (NULL == pui32FreeIndices)?psPageArrayData->uiTotalNumPages:ui32FreePageCount;
 590
 591         for (i = 0;i < ui32PagesToFree;i++)
 592         {
 593                 if(NULL == pui32FreeIndices)
 594                 {
 595                         ui32Index = i;
 596                 }
 597                 else
 598                 {
 599                         ui32Index = pui32FreeIndices[i];
 600                 }
 601
 602                 if (INVALID_PAGE != psPageArrayData->pasDevPAddr[ui32Index].uiAddr)
 603                 {
 604                         ui32PagesFreed++;
 605                         if (psPageArrayData->bPoisonOnFree)
 606                         {
 607                                 _PoisonAlloc(psPageArrayData->psDevNode,
 608                                                          &psPageArrayData->pasDevPAddr[ui32Index],
 609                                                          psPageArrayData->bFwLocalAlloc,
 610                                                          uiAllocSize,
 611                                                          _FreePoison,
 612                                                          _FreePoisonSize);
 613                         }
 614
 615                         RA_Free(pArena, psPageArrayData->pasDevPAddr[ui32Index].uiAddr);
 616
 617 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
 618 #if !defined(PVRSRV_ENABLE_MEMORY_STATS)
 619                         /* Allocation is done a page at a time */
 620                         PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, uiAllocSize);
 621 #else
 622                         {
 623                                 PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, psPageArrayData->pasDevPAddr[ui32Index].uiAddr);
 624                         }
 625 #endif
 626 #endif
 627                         psPageArrayData->pasDevPAddr[ui32Index].uiAddr = INVALID_PAGE;
 628                 }
 629         }
 630         psPageArrayData->iNumPagesAllocated -= ui32PagesFreed;
 631
 632     PVR_ASSERT(0 <= psPageArrayData->iNumPagesAllocated);
 633
 634         if(0 == psPageArrayData->iNumPagesAllocated)
 635         {
 636                 psPageArrayData->bHasLMPages = IMG_FALSE;
 637         }
 638
 639         PVR_DPF((PVR_DBG_MESSAGE, "%s: freed %d local memory for PMR @0x%p",__func__,(ui32PagesFreed*uiAllocSize), psPageArrayData));
 640         return PVRSRV_OK;
 641 }
 642
 643 /*
 644  *
 645  * Implementation of callback functions
 646  *
 647  */
 648
 649 /* destructor func is called after last reference disappears, but
 650    before PMR itself is freed. */
 651 static PVRSRV_ERROR
 652 PMRFinalizeLocalMem(PMR_IMPL_PRIVDATA pvPriv
 653                                  )
 654 {
 655         PVRSRV_ERROR eError;
 656         PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = NULL;
 657
 658         psLMAllocArrayData = pvPriv;
 659
 660         /*  We can't free pages until now. */
 661         if (psLMAllocArrayData->bHasLMPages)
 662         {
 663                 eError = _FreeLMPages(psLMAllocArrayData,NULL,0);
 664                 PVR_ASSERT (eError == PVRSRV_OK); /* can we do better? */
 665         }
 666
 667         eError = _FreeLMPageArray(psLMAllocArrayData);
 668         PVR_ASSERT (eError == PVRSRV_OK); /* can we do better? */
 669
 670         return PVRSRV_OK;
 671 }
 672
 673 /* callback function for locking the system physical page addresses.
 674    As we are LMA there is nothing to do as we control physical memory. */
 675 static PVRSRV_ERROR
 676 PMRLockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
 677 {
 678
 679     PVRSRV_ERROR eError;
 680     PMR_LMALLOCARRAY_DATA *psLMAllocArrayData;
 681
 682     psLMAllocArrayData = pvPriv;
 683
 684     if (psLMAllocArrayData->bOnDemand)
 685     {
 686                 /* Allocate Memory for deferred allocation */
 687                 eError = _AllocLMPages(psLMAllocArrayData, NULL);
 688         if (eError != PVRSRV_OK)
 689         {
 690                 return eError;
 691         }
 692     }
 693
 694         return PVRSRV_OK;
 695
 696 }
 697
 698 static PVRSRV_ERROR
 699 PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv
 700                                                            )
 701 {
 702     PVRSRV_ERROR eError = PVRSRV_OK;
 703     PMR_LMALLOCARRAY_DATA *psLMAllocArrayData;
 704
 705     psLMAllocArrayData = pvPriv;
 706
 707         if (psLMAllocArrayData->bOnDemand)
 708     {
 709                 /* Free Memory for deferred allocation */
 710                 eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
 711         if (eError != PVRSRV_OK)
 712         {
 713                 return eError;
 714         }
 715     }
 716
 717         PVR_ASSERT(eError == PVRSRV_OK);
 718         return eError;
 719 }
 720
 721 /* N.B.  It is assumed that PMRLockSysPhysAddressesLocalMem() is called _before_ this function! */
 722 static PVRSRV_ERROR
 723 PMRSysPhysAddrLocalMem(PMR_IMPL_PRIVDATA pvPriv,
 724                                            IMG_UINT32 ui32Log2PageSize,
 725                                            IMG_UINT32 ui32NumOfPages,
 726                                            IMG_DEVMEM_OFFSET_T *puiOffset,
 727                                            IMG_BOOL *pbValid,
 728                                            IMG_DEV_PHYADDR *psDevPAddr)
 729 {
 730         IMG_UINT32 idx;
 731         IMG_UINT32 uiLog2AllocSize;
 732         IMG_UINT32 uiNumAllocs;
 733         IMG_UINT64 uiAllocIndex;
 734         IMG_DEVMEM_OFFSET_T uiInAllocOffset;
 735         PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
 736
 737         if (psLMAllocArrayData->uiLog2AllocSize < ui32Log2PageSize)
 738         {
 739                 PVR_DPF((PVR_DBG_ERROR,
 740                          "%s: Requested physical addresses from PMR "
 741                          "for incompatible contiguity %u!",
 742                          __FUNCTION__,
 743                          ui32Log2PageSize));
 744                 return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
 745         }
 746
 747         uiNumAllocs = psLMAllocArrayData->uiTotalNumPages;
 748         if (uiNumAllocs > 1)
 749         {
 750                 PVR_ASSERT(psLMAllocArrayData->uiLog2AllocSize != 0);
 751                 uiLog2AllocSize = psLMAllocArrayData->uiLog2AllocSize;
 752
 753                 for (idx=0; idx < ui32NumOfPages; idx++)
 754                 {
 755                         if (pbValid[idx])
 756                         {
 757                                 uiAllocIndex = puiOffset[idx] >> uiLog2AllocSize;
 758                                 uiInAllocOffset = puiOffset[idx] - (uiAllocIndex << uiLog2AllocSize);
 759
 760                                 PVR_ASSERT(uiAllocIndex < uiNumAllocs);
 761                                 PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2AllocSize));
 762
 763                                 psDevPAddr[idx].uiAddr = psLMAllocArrayData->pasDevPAddr[uiAllocIndex].uiAddr + uiInAllocOffset;
 764                         }
 765                 }
 766         }
 767         else
 768         {
 769                 for (idx=0; idx < ui32NumOfPages; idx++)
 770                 {
 771                         if (pbValid[idx])
 772                         {
 773                                 psDevPAddr[idx].uiAddr = psLMAllocArrayData->pasDevPAddr[0].uiAddr + puiOffset[idx];
 774                         }
 775                 }
 776         }
 777
 778         return PVRSRV_OK;
 779 }
 780
 781 static PVRSRV_ERROR
 782 PMRAcquireKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
 783                                                                  size_t uiOffset,
 784                                                                  size_t uiSize,
 785                                                                  void **ppvKernelAddressOut,
 786                                                                  IMG_HANDLE *phHandleOut,
 787                                                                  PMR_FLAGS_T ulFlags)
 788 {
 789         PVRSRV_ERROR eError;
 790         PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = NULL;
 791         void *pvKernLinAddr = NULL;
 792         IMG_UINT32 ui32PageIndex = 0;
 793         size_t uiOffsetMask = uiOffset;
 794
 795         psLMAllocArrayData = pvPriv;
 796
 797         /* Check that we can map this in contiguously */
 798         if (psLMAllocArrayData->uiTotalNumPages != 1)
 799         {
 800                 size_t uiStart = uiOffset;
 801                 size_t uiEnd = uiOffset + uiSize - 1;
 802                 size_t uiPageMask = ~((1 << psLMAllocArrayData->uiLog2AllocSize) - 1);
 803
 804                 /* We can still map if only one page is required */
 805                 if ((uiStart & uiPageMask) != (uiEnd & uiPageMask))
 806                 {
 807                         eError = PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
 808                         goto e0;
 809                 }
 810
 811                 /* Locate the desired physical page to map in */
 812                 ui32PageIndex = uiOffset >> psLMAllocArrayData->uiLog2AllocSize;
 813                 uiOffsetMask = (1U << psLMAllocArrayData->uiLog2AllocSize) - 1;
 814         }
 815
 816         PVR_ASSERT(ui32PageIndex < psLMAllocArrayData->uiTotalNumPages);
 817
 818         eError = _MapAlloc(psLMAllocArrayData->psDevNode,
 819                                                 &psLMAllocArrayData->pasDevPAddr[ui32PageIndex],
 820                                                 psLMAllocArrayData->uiAllocSize,
 821                                                 psLMAllocArrayData->bFwLocalAlloc,
 822                                                 ulFlags,
 823                                                 &pvKernLinAddr);
 824
 825         *ppvKernelAddressOut = ((IMG_CHAR *) pvKernLinAddr) + (uiOffset & uiOffsetMask);
 826         *phHandleOut = pvKernLinAddr;
 827
 828         return eError;
 829
 830         /*
 831           error exit paths follow
 832         */
 833
 834  e0:
 835         PVR_ASSERT(eError != PVRSRV_OK);
 836         return eError;
 837 }
 838
 839 static void PMRReleaseKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
 840                                                                                                  IMG_HANDLE hHandle)
 841 {
 842         PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = NULL;
 843         void *pvKernLinAddr = NULL;
 844
 845         psLMAllocArrayData = (PMR_LMALLOCARRAY_DATA *) pvPriv;
 846         pvKernLinAddr = (void *) hHandle;
 847
 848         _UnMapAlloc(psLMAllocArrayData->psDevNode,
 849                                 psLMAllocArrayData->uiAllocSize,
 850                                 psLMAllocArrayData->bFwLocalAlloc,
 851                                 0,
 852                                 pvKernLinAddr);
 853 }
 854
 855
 856 static PVRSRV_ERROR
 857 CopyBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
 858                                   IMG_DEVMEM_OFFSET_T uiOffset,
 859                                   IMG_UINT8 *pcBuffer,
 860                                   size_t uiBufSz,
 861                                   size_t *puiNumBytes,
 862                                   void (*pfnCopyBytes)(IMG_UINT8 *pcBuffer,
 863                                                                            IMG_UINT8 *pcPMR,
 864                                                                            size_t uiSize))
 865 {
 866         PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = NULL;
 867         size_t uiBytesCopied;
 868         size_t uiBytesToCopy;
 869         size_t uiBytesCopyableFromAlloc;
 870         void *pvMapping = NULL;
 871         IMG_UINT8 *pcKernelPointer = NULL;
 872         size_t uiBufferOffset;
 873         IMG_UINT64 uiAllocIndex;
 874         IMG_DEVMEM_OFFSET_T uiInAllocOffset;
 875         PVRSRV_ERROR eError;
 876
 877         psLMAllocArrayData = pvPriv;
 878
 879         uiBytesCopied = 0;
 880         uiBytesToCopy = uiBufSz;
 881         uiBufferOffset = 0;
 882
 883         if (psLMAllocArrayData->uiTotalNumPages > 1)
 884         {
 885                 while (uiBytesToCopy > 0)
 886                 {
 887                         /* we have to map one alloc in at a time */
 888                         PVR_ASSERT(psLMAllocArrayData->uiLog2AllocSize != 0);
 889                         uiAllocIndex = uiOffset >> psLMAllocArrayData->uiLog2AllocSize;
 890                         uiInAllocOffset = uiOffset - (uiAllocIndex << psLMAllocArrayData->uiLog2AllocSize);
 891                         uiBytesCopyableFromAlloc = uiBytesToCopy;
 892                         if (uiBytesCopyableFromAlloc + uiInAllocOffset > (1ULL << psLMAllocArrayData->uiLog2AllocSize))
 893                         {
 894                                 uiBytesCopyableFromAlloc = TRUNCATE_64BITS_TO_SIZE_T((1ULL << psLMAllocArrayData->uiLog2AllocSize)-uiInAllocOffset);
 895                         }
 896
 897                         PVR_ASSERT(uiBytesCopyableFromAlloc != 0);
 898                         PVR_ASSERT(uiAllocIndex < psLMAllocArrayData->uiTotalNumPages);
 899                         PVR_ASSERT(uiInAllocOffset < (1ULL << psLMAllocArrayData->uiLog2AllocSize));
 900
 901                         eError = _MapAlloc(psLMAllocArrayData->psDevNode,
 902                                                                 &psLMAllocArrayData->pasDevPAddr[uiAllocIndex],
 903                                                                 psLMAllocArrayData->uiAllocSize,
 904                                                                 psLMAllocArrayData->bFwLocalAlloc,
 905                                                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
 906                                                                 &pvMapping);
 907                         if (eError != PVRSRV_OK)
 908                         {
 909                                 goto e0;
 910                         }
 911                         pcKernelPointer = pvMapping;
 912                         pfnCopyBytes(&pcBuffer[uiBufferOffset], &pcKernelPointer[uiInAllocOffset], uiBytesCopyableFromAlloc);
 913
 914                         _UnMapAlloc(psLMAllocArrayData->psDevNode,
 915                                                 psLMAllocArrayData->uiAllocSize,
 916                                                 psLMAllocArrayData->bFwLocalAlloc,
 917                                                 0,
 918                                                 pvMapping);
 919
 920                         uiBufferOffset += uiBytesCopyableFromAlloc;
 921                         uiBytesToCopy -= uiBytesCopyableFromAlloc;
 922                         uiOffset += uiBytesCopyableFromAlloc;
 923                         uiBytesCopied += uiBytesCopyableFromAlloc;
 924                 }
 925         }
 926         else
 927         {
 928                         PVR_ASSERT((uiOffset + uiBufSz) <= psLMAllocArrayData->uiAllocSize);
 929                         PVR_ASSERT(psLMAllocArrayData->uiAllocSize != 0);
 930                         eError = _MapAlloc(psLMAllocArrayData->psDevNode,
 931                                                                 &psLMAllocArrayData->pasDevPAddr[0],
 932                                                                 psLMAllocArrayData->uiAllocSize,
 933                                                                 psLMAllocArrayData->bFwLocalAlloc,
 934                                                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
 935                                                                 &pvMapping);
 936                         if (eError != PVRSRV_OK)
 937                         {
 938                                 goto e0;
 939                         }
 940                         pcKernelPointer = pvMapping;
 941                         pfnCopyBytes(pcBuffer, &pcKernelPointer[uiOffset], uiBufSz);
 942
 943                         _UnMapAlloc(psLMAllocArrayData->psDevNode,
 944                                                 psLMAllocArrayData->uiAllocSize,
 945                                                 psLMAllocArrayData->bFwLocalAlloc,
 946                                                 0,
 947                                                 pvMapping);
 948
 949                         uiBytesCopied = uiBufSz;
 950         }
 951         *puiNumBytes = uiBytesCopied;
 952         return PVRSRV_OK;
 953 e0:
 954         *puiNumBytes = uiBytesCopied;
 955         return eError;
 956 }
 957
 958 static void ReadLocalMem(IMG_UINT8 *pcBuffer,
 959                                                  IMG_UINT8 *pcPMR,
 960                                                  size_t uiSize)
 961 {
 962         /* NOTE: 'CachedMemCopy' means the operating system default memcpy, which
 963          *       we *assume* in the LMA code will be faster, and doesn't need to
 964          *       worry about ARM64.
 965          */
 966         OSCachedMemCopy(pcBuffer, pcPMR, uiSize);
 967 }
 968
 969 static PVRSRV_ERROR
 970 PMRReadBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
 971                                   IMG_DEVMEM_OFFSET_T uiOffset,
 972                                   IMG_UINT8 *pcBuffer,
 973                                   size_t uiBufSz,
 974                                   size_t *puiNumBytes)
 975 {
 976         return CopyBytesLocalMem(pvPriv,
 977                                                          uiOffset,
 978                                                          pcBuffer,
 979                                                          uiBufSz,
 980                                                          puiNumBytes,
 981                                                          ReadLocalMem);
 982 }
 983
 984 static void WriteLocalMem(IMG_UINT8 *pcBuffer,
 985                                                   IMG_UINT8 *pcPMR,
 986                                                   size_t uiSize)
 987 {
 988         /* NOTE: 'CachedMemCopy' means the operating system default memcpy, which
 989          *       we *assume* in the LMA code will be faster, and doesn't need to
 990          *       worry about ARM64.
 991          */
 992         OSCachedMemCopy(pcPMR, pcBuffer, uiSize);
 993 }
 994
 995 static PVRSRV_ERROR
 996 PMRWriteBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
 997                                           IMG_DEVMEM_OFFSET_T uiOffset,
 998                                           IMG_UINT8 *pcBuffer,
 999                                           size_t uiBufSz,
1000                                           size_t *puiNumBytes)
1001 {
1002         return CopyBytesLocalMem(pvPriv,
1003                                                          uiOffset,
1004                                                          pcBuffer,
1005                                                          uiBufSz,
1006                                                          puiNumBytes,
1007                                                          WriteLocalMem);
1008 }
1009
1010 /*************************************************************************/ /*!
1011 @Function       PMRChangeSparseMemLocalMem
1012 @Description    This function Changes the sparse mapping by allocating & freeing
1013                                 of pages. It does also change the GPU maps accordingly
1014 @Return         PVRSRV_ERROR failure code
1015 */ /**************************************************************************/
1016 static PVRSRV_ERROR
1017 PMRChangeSparseMemLocalMem(PMR_IMPL_PRIVDATA pPriv,
1018                            const PMR *psPMR,
1019                            IMG_UINT32 ui32AllocPageCount,
1020                            IMG_UINT32 *pai32AllocIndices,
1021                            IMG_UINT32 ui32FreePageCount,
1022                            IMG_UINT32 *pai32FreeIndices,
1023                            IMG_UINT32 uiFlags)
1024 {
1025         PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
1026
1027         IMG_UINT32 ui32AdtnlAllocPages = 0;
1028         IMG_UINT32 ui32AdtnlFreePages = 0;
1029         IMG_UINT32 ui32CommonRequstCount = 0;
1030         IMG_UINT32 ui32Loop = 0;
1031         IMG_UINT32 ui32Index = 0;
1032         IMG_UINT32 uiAllocpgidx;
1033         IMG_UINT32 uiFreepgidx;
1034
1035         PMR_LMALLOCARRAY_DATA *psPMRPageArrayData = (PMR_LMALLOCARRAY_DATA *)pPriv;
1036         IMG_DEV_PHYADDR sPhyAddr;
1037
1038 #if defined(DEBUG)
1039         IMG_BOOL bPoisonFail = IMG_FALSE;
1040         IMG_BOOL bZeroFail = IMG_FALSE;
1041 #endif
1042
1043         /* Fetch the Page table array represented by the PMR */
1044         IMG_DEV_PHYADDR *psPageArray = psPMRPageArrayData->pasDevPAddr;
1045         PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappigTable(psPMR);
1046
1047         /* The incoming request is classified into two operations independent of
1048          * each other: alloc & free pages.
1049          * These operations can be combined with two mapping operations as well
1050          * which are GPU & CPU space mappings.
1051          *
1052          * From the alloc and free page requests, the net amount of pages to be
1053          * allocated or freed is computed. Pages that were requested to be freed
1054          * will be reused to fulfil alloc requests.
1055          *
1056          * The order of operations is:
1057          * 1. Allocate new pages from the OS
1058          * 2. Move the free pages from free request to alloc positions.
1059          * 3. Free the rest of the pages not used for alloc
1060          *
1061          * Alloc parameters are validated at the time of allocation
1062          * and any error will be handled then. */
1063
1064         if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
1065         {
1066                 ui32CommonRequstCount = (ui32AllocPageCount > ui32FreePageCount) ?
1067                                 ui32FreePageCount : ui32AllocPageCount;
1068
1069                 PDUMP_PANIC(SPARSEMEM_SWAP, "Request to swap alloc & free pages not supported");
1070         }
1071
1072         if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
1073         {
1074                 ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequstCount;
1075         }
1076         else
1077         {
1078                 ui32AllocPageCount = 0;
1079         }
1080
1081         if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
1082         {
1083                 ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequstCount;
1084         }
1085         else
1086         {
1087                 ui32FreePageCount = 0;
1088         }
1089
1090         if (0 == (ui32CommonRequstCount || ui32AdtnlAllocPages || ui32AdtnlFreePages))
1091         {
1092                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1093                 return eError;
1094         }
1095
1096         {
1097                 /* Validate the free page indices */
1098                 if (ui32FreePageCount)
1099                 {
1100                         if (NULL != pai32FreeIndices)
1101                         {
1102                                 for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
1103                                 {
1104                                         uiFreepgidx = pai32FreeIndices[ui32Loop];
1105
1106                                         if (uiFreepgidx > psPMRPageArrayData->uiTotalNumPages)
1107                                         {
1108                                                 eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
1109                                                 goto e0;
1110                                         }
1111
1112                                         if (INVALID_PAGE == psPageArray[uiFreepgidx].uiAddr)
1113                                         {
1114                                                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1115                                                 goto e0;
1116                                         }
1117                                 }
1118                         }else{
1119                                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1120                                 return eError;
1121                         }
1122                 }
1123
1124                 /*The following block of code verifies any issues with common alloc page indices */
1125                 for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
1126                 {
1127                         uiAllocpgidx = pai32AllocIndices[ui32Loop];
1128                         if (uiAllocpgidx > psPMRPageArrayData->uiTotalNumPages)
1129                         {
1130                                 eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
1131                                 goto e0;
1132                         }
1133
1134                         if (SPARSE_REMAP_MEM != (uiFlags & SPARSE_REMAP_MEM))
1135                         {
1136                                 if ((INVALID_PAGE != psPageArray[uiAllocpgidx].uiAddr) ||
1137                                                 (TRANSLATION_INVALID != psPMRMapTable->aui32Translation[uiAllocpgidx]))
1138                                 {
1139                                         eError = PVRSRV_ERROR_INVALID_PARAMS;
1140                                         goto e0;
1141                                 }
1142                         }
1143                         else
1144                         {
1145                                 if ((INVALID_PAGE ==  psPageArray[uiAllocpgidx].uiAddr) ||
1146                                     (TRANSLATION_INVALID == psPMRMapTable->aui32Translation[uiAllocpgidx]))
1147                                 {
1148                                         eError = PVRSRV_ERROR_INVALID_PARAMS;
1149                                         goto e0;
1150                                 }
1151                         }
1152                 }
1153
1154
1155                 ui32Loop = 0;
1156
1157                 /* Allocate new pages */
1158                 if (0 != ui32AdtnlAllocPages)
1159                 {
1160                         /* Say how many pages to allocate */
1161                         psPMRPageArrayData->uiPagesToAlloc = ui32AdtnlAllocPages;
1162
1163                         eError = _AllocLMPages(psPMRPageArrayData, pai32AllocIndices);
1164                         if (PVRSRV_OK != eError)
1165                         {
1166                                 PVR_DPF((PVR_DBG_ERROR,
1167                                          "%s: New Addtl Allocation of pages failed",
1168                                          __FUNCTION__));
1169                                 goto e0;
1170                         }
1171
1172                         /* Mark the corresponding pages of translation table as valid */
1173                         for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
1174                         {
1175                                 psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
1176                         }
1177                 }
1178
1179                 ui32Index = ui32Loop;
1180
1181                 /* Move the corresponding free pages to alloc request */
1182                 for (ui32Loop = 0; ui32Loop < ui32CommonRequstCount; ui32Loop++, ui32Index++)
1183                 {
1184
1185                         uiAllocpgidx = pai32AllocIndices[ui32Index];
1186                         uiFreepgidx =  pai32FreeIndices[ui32Loop];
1187                         sPhyAddr = psPageArray[uiAllocpgidx];
1188                         psPageArray[uiAllocpgidx] = psPageArray[uiFreepgidx];
1189
1190                         /* Is remap mem used in real world scenario? Should it be turned to a
1191                          *  debug feature? The condition check needs to be out of loop, will be
1192                          *  done at later point though after some analysis */
1193                         if (SPARSE_REMAP_MEM != (uiFlags & SPARSE_REMAP_MEM))
1194                         {
1195                                 psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
1196                                 psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
1197                                 psPageArray[uiFreepgidx].uiAddr = INVALID_PAGE;
1198                         }
1199                         else
1200                         {
1201                                 psPageArray[uiFreepgidx] = sPhyAddr;
1202                                 psPMRMapTable->aui32Translation[uiFreepgidx] = uiFreepgidx;
1203                                 psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
1204                         }
1205
1206                         /* Be sure to honour the attributes associated with the allocation
1207                          * such as zeroing, poisoning etc. */
1208                         if (psPMRPageArrayData->bPoisonOnAlloc)
1209                         {
1210                                 eError = _PoisonAlloc(psPMRPageArrayData->psDevNode,
1211                                                       &psPMRPageArrayData->pasDevPAddr[uiAllocpgidx],
1212                                                       psPMRPageArrayData->bFwLocalAlloc,
1213                                                       psPMRPageArrayData->uiAllocSize,
1214                                                       _AllocPoison,
1215                                                       _AllocPoisonSize);
1216
1217                                 /* Consider this as a soft failure and go ahead but log error to kernel log */
1218                                 if (eError != PVRSRV_OK)
1219                                 {
1220 #if defined(DEBUG)
1221                                         bPoisonFail = IMG_TRUE;
1222 #endif
1223                                 }
1224                         }
1225                         else
1226                         {
1227                                 if (psPMRPageArrayData->bZeroOnAlloc)
1228                                 {
1229                                         eError = _ZeroAlloc(psPMRPageArrayData->psDevNode,
1230                                                             &psPMRPageArrayData->pasDevPAddr[uiAllocpgidx],
1231                                                             psPMRPageArrayData->bFwLocalAlloc,
1232                                                             psPMRPageArrayData->uiAllocSize);
1233                                         /* Consider this as a soft failure and go ahead but log error to kernel log */
1234                                         if (eError != PVRSRV_OK)
1235                                         {
1236 #if defined(DEBUG)
1237                                                 /*Don't think we need to zero  any pages further*/
1238                                                 bZeroFail = IMG_TRUE;
1239 #endif
1240                                         }
1241                                 }
1242                         }
1243                 }
1244
1245                 /*Free the additional free pages */
1246                 if (0 != ui32AdtnlFreePages)
1247                 {
1248                         ui32Index = ui32Loop;
1249                         _FreeLMPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages);
1250                         ui32Loop = 0;
1251
1252                         while(ui32Loop++ < ui32AdtnlFreePages)
1253                         {
1254                                 /*Set the corresponding mapping table entry to invalid address */
1255                                 psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Index++]] = TRANSLATION_INVALID;
1256                         }
1257                 }
1258
1259         }
1260
1261 #if defined(DEBUG)
1262         if(IMG_TRUE == bPoisonFail)
1263         {
1264                 PVR_DPF((PVR_DBG_ERROR, "%s: Error in poisoning the page", __FUNCTION__));
1265         }
1266
1267         if(IMG_TRUE == bZeroFail)
1268         {
1269                 PVR_DPF((PVR_DBG_ERROR, "%s: Error in zeroing the page", __FUNCTION__));
1270         }
1271 #endif
1272
1273         /* Update the PMR memory holding information */
1274         eError = PVRSRV_OK;
1275
1276 e0:
1277                 return eError;
1278
1279 }
1280
1281 /*************************************************************************/ /*!
1282 @Function       PMRChangeSparseMemCPUMapLocalMem
1283 @Description    This function Changes CPU maps accordingly
1284 @Return         PVRSRV_ERROR failure code
1285 */ /**************************************************************************/
1286 static
1287 PVRSRV_ERROR PMRChangeSparseMemCPUMapLocalMem(PMR_IMPL_PRIVDATA pPriv,
1288                                               const PMR *psPMR,
1289                                               IMG_UINT64 sCpuVAddrBase,
1290                                               IMG_UINT32 ui32AllocPageCount,
1291                                               IMG_UINT32 *pai32AllocIndices,
1292                                               IMG_UINT32 ui32FreePageCount,
1293                                               IMG_UINT32 *pai32FreeIndices)
1294 {
1295         IMG_DEV_PHYADDR *psPageArray;
1296         PMR_LMALLOCARRAY_DATA *psPMRPageArrayData = (PMR_LMALLOCARRAY_DATA *)pPriv;
1297         uintptr_t sCpuVABase = sCpuVAddrBase;
1298         IMG_CPU_PHYADDR sCpuAddrPtr;
1299         IMG_BOOL bValid;
1300
1301         /*Get the base address of the heap */
1302         PMR_CpuPhysAddr(psPMR,
1303                         psPMRPageArrayData->uiLog2AllocSize,
1304                         1,
1305                         0,      /* offset zero here mean first page in the PMR */
1306                         &sCpuAddrPtr,
1307                         &bValid);
1308
1309         /* Phys address of heap is computed here by subtracting the offset of this page
1310          * basically phys address of any page = Base address of heap + offset of the page */
1311         sCpuAddrPtr.uiAddr -= psPMRPageArrayData->pasDevPAddr[0].uiAddr;
1312         psPageArray = psPMRPageArrayData->pasDevPAddr;
1313
1314         return OSChangeSparseMemCPUAddrMap((void **)psPageArray,
1315                                            sCpuVABase,
1316                                            sCpuAddrPtr,
1317                                            ui32AllocPageCount,
1318                                            pai32AllocIndices,
1319                                            ui32FreePageCount,
1320                                            pai32FreeIndices,
1321                                            IMG_TRUE);
1322 }
1323
1324
1325 static PMR_IMPL_FUNCTAB _sPMRLMAFuncTab = {
1326         /* pfnLockPhysAddresses */
1327         &PMRLockSysPhysAddressesLocalMem,
1328         /* pfnUnlockPhysAddresses */
1329         &PMRUnlockSysPhysAddressesLocalMem,
1330         /* pfnDevPhysAddr */
1331         &PMRSysPhysAddrLocalMem,
1332         /* pfnAcquireKernelMappingData */
1333         &PMRAcquireKernelMappingDataLocalMem,
1334         /* pfnReleaseKernelMappingData */
1335         &PMRReleaseKernelMappingDataLocalMem,
1336 #if defined(INTEGRITY_OS)
1337         /* pfnMapMemoryObject */
1338     NULL,
1339         /* pfnUnmapMemoryObject */
1340     NULL,
1341 #endif
1342         /* pfnReadBytes */
1343         &PMRReadBytesLocalMem,
1344         /* pfnWriteBytes */
1345         &PMRWriteBytesLocalMem,
1346         /* .pfnUnpinMem */
1347         NULL,
1348         /* .pfnPinMem */
1349         NULL,
1350         /* pfnChangeSparseMem*/
1351         &PMRChangeSparseMemLocalMem,
1352         /* pfnChangeSparseMemCPUMap */
1353         &PMRChangeSparseMemCPUMapLocalMem,
1354         /* pfnMMap */
1355         NULL,
1356         /* pfnFinalize */
1357         &PMRFinalizeLocalMem
1358 };
1359
1360 PVRSRV_ERROR
1361 PhysmemNewLocalRamBackedPMR(PVRSRV_DEVICE_NODE *psDevNode,
1362                                                         IMG_DEVMEM_SIZE_T uiSize,
1363                                                         IMG_DEVMEM_SIZE_T uiChunkSize,
1364                                                         IMG_UINT32 ui32NumPhysChunks,
1365                                                         IMG_UINT32 ui32NumVirtChunks,
1366                                                         IMG_UINT32 *pui32MappingTable,
1367                                                         IMG_UINT32 uiLog2PageSize,
1368                                                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
1369                                                         const IMG_CHAR *pszAnnotation,
1370                                                         PMR **ppsPMRPtr)
1371 {
1372         PVRSRV_ERROR eError;
1373         PVRSRV_ERROR eError2;
1374         PMR *psPMR = NULL;
1375         PMR_LMALLOCARRAY_DATA *psPrivData = NULL;
1376         PMR_FLAGS_T uiPMRFlags;
1377         PHYS_HEAP *psPhysHeap;
1378         IMG_BOOL bZero;
1379         IMG_BOOL bPoisonOnAlloc;
1380         IMG_BOOL bPoisonOnFree;
1381         IMG_BOOL bOnDemand;
1382         IMG_BOOL bContig;
1383         IMG_BOOL bFwLocalAlloc;
1384         IMG_BOOL bCpuLocalAlloc;
1385
1386         if (PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags) &&
1387                 (ui32NumPhysChunks == ui32NumVirtChunks))
1388         {
1389                 bContig = IMG_TRUE;
1390         }
1391         else
1392         {
1393                 bContig = IMG_FALSE;
1394         }
1395
1396         bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiFlags) ? IMG_TRUE : IMG_FALSE;
1397         bFwLocalAlloc = PVRSRV_CHECK_FW_LOCAL(uiFlags) ? IMG_TRUE : IMG_FALSE;
1398         bCpuLocalAlloc = PVRSRV_CHECK_CPU_LOCAL(uiFlags) ? IMG_TRUE : IMG_FALSE;
1399         bZero = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags) ? IMG_TRUE : IMG_FALSE;
1400         bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags) ? IMG_TRUE : IMG_FALSE;
1401         bPoisonOnFree = PVRSRV_CHECK_POISON_ON_FREE(uiFlags) ? IMG_TRUE : IMG_FALSE;
1402
1403         if (bZero && bPoisonOnAlloc)
1404         {
1405                 /* Zero on Alloc and Poison on Alloc are mutually exclusive */
1406                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1407                 goto errorOnParam;
1408         }
1409
1410         /* Silently round up alignment/pagesize if request was less that
1411            PAGE_SHIFT, because it would never be harmful for memory to be
1412            _more_ contiguous that was desired */
1413
1414         uiLog2PageSize = OSGetPageShift() > uiLog2PageSize
1415                 ? OSGetPageShift()
1416                 : uiLog2PageSize;
1417
1418         /* In case we have a non-sparse allocation tolerate bad requests and round up.
1419          * For sparse allocations the users have to make sure to meet the right
1420          * requirements. */
1421         if (ui32NumPhysChunks == ui32NumVirtChunks &&
1422                 ui32NumVirtChunks == 1)
1423         {
1424                 /* Round up allocation size to at least a full OSGetPageSize() */
1425                 uiSize = PVR_ALIGN(uiSize, OSGetPageSize());
1426                 uiChunkSize = uiSize;
1427         }
1428
1429         if (bFwLocalAlloc)
1430         {
1431                 psPhysHeap = psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_FW_LOCAL];
1432         }
1433         else if (bCpuLocalAlloc)
1434         {
1435                 psPhysHeap = psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_CPU_LOCAL];
1436         }
1437         else
1438         {
1439                 psPhysHeap = psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_GPU_LOCAL];
1440         }
1441
1442         /* Create Array structure that holds the physical pages */
1443         eError = _AllocLMPageArray(psDevNode,
1444                                    uiChunkSize * ui32NumVirtChunks,
1445                                    uiChunkSize,
1446                                    ui32NumPhysChunks,
1447                                    ui32NumVirtChunks,
1448                                    pui32MappingTable,
1449                                    uiLog2PageSize,
1450                                    bZero,
1451                                    bPoisonOnAlloc,
1452                                    bPoisonOnFree,
1453                                    bContig,
1454                                    bOnDemand,
1455                                    bFwLocalAlloc,
1456                                    psPhysHeap,
1457                                    uiFlags,
1458                                    &psPrivData);
1459         if (eError != PVRSRV_OK)
1460         {
1461                 goto errorOnAllocPageArray;
1462         }
1463
1464
1465         if (!bOnDemand)
1466         {
1467                 /* Allocate the physical pages */
1468                 eError = _AllocLMPages(psPrivData,pui32MappingTable);
1469                 if (eError != PVRSRV_OK)
1470                 {
1471                         goto errorOnAllocPages;
1472                 }
1473         }
1474
1475         /* In this instance, we simply pass flags straight through.
1476
1477            Generically, uiFlags can include things that control the PMR
1478            factory, but we don't need any such thing (at the time of
1479            writing!), and our caller specifies all PMR flags so we don't
1480            need to meddle with what was given to us.
1481         */
1482         uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
1483         /* check no significant bits were lost in cast due to different
1484            bit widths for flags */
1485         PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
1486
1487     if (bOnDemand)
1488     {
1489         PDUMPCOMMENT("Deferred Allocation PMR (LMA)");
1490     }
1491
1492
1493         eError = PMRCreatePMR(psDevNode,
1494                                                   psPhysHeap,
1495                                                   uiSize,
1496                           uiChunkSize,
1497                           ui32NumPhysChunks,
1498                           ui32NumVirtChunks,
1499                           pui32MappingTable,
1500                                                   uiLog2PageSize,
1501                                                   uiPMRFlags,
1502                                                   pszAnnotation,
1503                                                   &_sPMRLMAFuncTab,
1504                                                   psPrivData,
1505                                                   PMR_TYPE_LMA,
1506                                                   &psPMR,
1507                                                   IMG_FALSE);
1508         if (eError != PVRSRV_OK)
1509         {
1510                 PVR_DPF((PVR_DBG_ERROR, "PhysmemNewLocalRamBackedPMR: Unable to create PMR (status=%d)", eError));
1511                 goto errorOnCreate;
1512         }
1513
1514         *ppsPMRPtr = psPMR;
1515         return PVRSRV_OK;
1516
1517 errorOnCreate:
1518         if(!bOnDemand && psPrivData->bHasLMPages)
1519         {
1520                 eError2 = _FreeLMPages(psPrivData, NULL,0);
1521                 PVR_ASSERT(eError2 == PVRSRV_OK);
1522         }
1523
1524 errorOnAllocPages:
1525         eError2 = _FreeLMPageArray(psPrivData);
1526         PVR_ASSERT(eError2 == PVRSRV_OK);
1527
1528 errorOnAllocPageArray:
1529 errorOnParam:
1530         PVR_ASSERT(eError != PVRSRV_OK);
1531         return eError;
1532 }
1533
1534 #if defined(SUPPORT_GPUVIRT_VALIDATION)
1535
1536 struct PidOSidCouplingList
1537 {
1538         IMG_PID     pId;
1539         IMG_UINT32  ui32OSid;
1540         IMG_UINT32      ui32OSidReg;
1541     IMG_BOOL    bOSidAxiProt;
1542
1543         struct PidOSidCouplingList *psNext;
1544 };
1545 typedef struct PidOSidCouplingList PidOSidCouplingList;
1546
1547 static PidOSidCouplingList *psPidOSidHead=NULL;
1548 static PidOSidCouplingList *psPidOSidTail=NULL;
1549
1550 void InsertPidOSidsCoupling(IMG_PID pId, IMG_UINT32 ui32OSid, IMG_UINT32 ui32OSidReg, IMG_BOOL bOSidAxiProt)
1551 {
1552         PidOSidCouplingList *psTmp;
1553
1554     PVR_DPF((PVR_DBG_MESSAGE,"(GPU Virtualization Validation): Inserting (PID/ OSid/ OSidReg/ IsSecure) (%d/ %d/ %d/ %s) into list",
1555                  pId,ui32OSid, ui32OSidReg, (bOSidAxiProt)?"Yes":"No"));
1556
1557         psTmp=OSAllocMem(sizeof(PidOSidCouplingList));
1558
1559         if (psTmp==NULL)
1560         {
1561                 PVR_DPF((PVR_DBG_ERROR,"(GPU Virtualization Validation): Memory allocation failed. No list insertion => program will execute normally.\n"));
1562                 return ;
1563         }
1564
1565         psTmp->pId=pId;
1566         psTmp->ui32OSid=ui32OSid;
1567         psTmp->ui32OSidReg=ui32OSidReg;
1568     psTmp->bOSidAxiProt = bOSidAxiProt;
1569
1570         psTmp->psNext=NULL;
1571         if (psPidOSidHead==NULL)
1572         {
1573                 psPidOSidHead=psTmp;
1574                 psPidOSidTail=psTmp;
1575         }
1576         else
1577         {
1578                 psPidOSidTail->psNext=psTmp;
1579                 psPidOSidTail=psTmp;
1580         }
1581
1582         return ;
1583 }
1584
1585 void RetrieveOSidsfromPidList(IMG_PID pId, IMG_UINT32 *pui32OSid, IMG_UINT32 *pui32OSidReg, IMG_BOOL *pbOSidAxiProt)
1586 {
1587         PidOSidCouplingList *psTmp;
1588
1589         for (psTmp=psPidOSidHead;psTmp!=NULL;psTmp=psTmp->psNext)
1590         {
1591                 if (psTmp->pId==pId)
1592                 {
1593             (*pui32OSid)     = psTmp->ui32OSid;
1594             (*pui32OSidReg)  = psTmp->ui32OSidReg;
1595             (*pbOSidAxiProt) = psTmp->bOSidAxiProt;
1596
1597                         return ;
1598                 }
1599         }
1600
1601         (*pui32OSid)=0;
1602         (*pui32OSidReg)=0;
1603     (*pbOSidAxiProt) = IMG_FALSE;
1604
1605         return ;
1606 }
1607
1608 void RemovePidOSidCoupling(IMG_PID pId)
1609 {
1610         PidOSidCouplingList *psTmp, *psPrev=NULL;
1611
1612         for (psTmp=psPidOSidHead; psTmp!=NULL; psTmp=psTmp->psNext)
1613         {
1614                 if (psTmp->pId==pId) break;
1615                 psPrev=psTmp;
1616         }
1617
1618         if (psTmp==NULL)
1619         {
1620                 return ;
1621         }
1622
1623         PVR_DPF((PVR_DBG_MESSAGE,"(GPU Virtualization Validation): Deleting Pairing %d / (%d - %d) from list",psTmp->pId, psTmp->ui32OSid, psTmp->ui32OSidReg));
1624
1625         if (psTmp==psPidOSidHead)
1626         {
1627                 if (psPidOSidHead->psNext==NULL)
1628                 {
1629                         psPidOSidHead=NULL;
1630                         psPidOSidTail=NULL;
1631                         OSFreeMem(psTmp);
1632
1633                         return ;
1634                 }
1635
1636                 psPidOSidHead=psPidOSidHead->psNext;
1637                 OSFreeMem(psTmp);
1638                 return ;
1639         }
1640
1641         if (psPrev==NULL) return ;
1642
1643         psPrev->psNext=psTmp->psNext;
1644         if (psTmp==psPidOSidTail)
1645         {
1646                 psPidOSidTail=psPrev;
1647         }
1648
1649         OSFreeMem(psTmp);
1650
1651         return ;
1652 }
1653
1654 #endif
1655