drivers/staging/imgtec/rogue/pmr_os.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          Linux OS PMR functions
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @License        Dual MIT/GPLv2
   6
   7 The contents of this file are subject to the MIT license as set out below.
   8
   9 Permission is hereby granted, free of charge, to any person obtaining a copy
  10 of this software and associated documentation files (the "Software"), to deal
  11 in the Software without restriction, including without limitation the rights
  12 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  13 copies of the Software, and to permit persons to whom the Software is
  14 furnished to do so, subject to the following conditions:
  15
  16 The above copyright notice and this permission notice shall be included in
  17 all copies or substantial portions of the Software.
  18
  19 Alternatively, the contents of this file may be used under the terms of
  20 the GNU General Public License Version 2 ("GPL") in which case the provisions
  21 of GPL are applicable instead of those above.
  22
  23 If you wish to allow use of your version of this file only under the terms of
  24 GPL, and not to allow others to use your version of this file under the terms
  25 of the MIT license, indicate your decision by deleting the provisions above
  26 and replace them with the notice and other provisions required by GPL as set
  27 out in the file called "GPL-COPYING" included in this distribution. If you do
  28 not delete the provisions above, a recipient may use your version of this file
  29 under the terms of either the MIT license or GPL.
  30
  31 This License is also included in this distribution in the file called
  32 "MIT-COPYING".
  33
  34 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  35 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  36 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  37 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  38 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  39 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  40 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  41 */ /**************************************************************************/
  42
  43 #include <asm/io.h>
  44 #include <asm/page.h>
  45 #include <linux/mm.h>
  46 #include <linux/dma-mapping.h>
  47 #if defined(CONFIG_L4)
  48 #include <asm/api-l4env/api.h>
  49 #endif
  50 #include <linux/version.h>
  51 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
  52 #include <linux/pfn_t.h>
  53 #include <linux/pfn.h>
  54 #endif
  55
  56 #include "img_defs.h"
  57 #include "pvr_debug.h"
  58 #include "allocmem.h"
  59 #include "devicemem_server_utils.h"
  60 #include "pmr.h"
  61 #include "pmr_os.h"
  62
  63 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
  64 #include "process_stats.h"
  65 #endif
  66
  67 #include "kernel_compatibility.h"
  68
  69 /*
  70  * x86_32:
  71  * Use vm_insert_page because remap_pfn_range has issues when mapping HIGHMEM
  72  * pages with default memory attributes; these HIGHMEM pages are skipped in
  73  * set_pages_array_[uc,wc] during allocation; see reserve_pfn_range().
  74  * Also vm_insert_page is faster.
  75  *
  76  * x86_64:
  77  * Use vm_insert_page because it is faster.
  78  *
  79  * Other platforms:
  80  * Use remap_pfn_range by default because it does not issue a cache flush.
  81  * It is known that ARM32 benefits from this. When other platforms become
  82  * available it has to be investigated if this assumption holds for them as well.
  83  *
  84  * Since vm_insert_page does more precise memory accounting we have the build
  85  * flag PVR_MMAP_USE_VM_INSERT that forces its use. This is useful as a debug
  86  * feature.
  87  *
  88  */
  89 #if defined(CONFIG_X86) || defined(PVR_MMAP_USE_VM_INSERT)
  90 #define PMR_OS_USE_VM_INSERT_PAGE 1
  91 #endif
  92
  93 static void MMapPMROpen(struct vm_area_struct *ps_vma)
  94 {
  95         PMR *psPMR = ps_vma->vm_private_data;
  96
  97         /* Our VM flags should ensure this function never gets called */
  98         PVR_DPF((PVR_DBG_WARNING,
  99                          "%s: Unexpected mmap open call, this is probably an application bug.",
 100                          __func__));
 101         PVR_DPF((PVR_DBG_WARNING,
 102                          "%s: vma struct: 0x%p, vAddr: %#lX, length: %#lX, PMR pointer: 0x%p",
 103                          __func__,
 104                          ps_vma,
 105                          ps_vma->vm_start,
 106                          ps_vma->vm_end - ps_vma->vm_start,
 107                          psPMR));
 108
 109         /* In case we get called anyway let's do things right by increasing the refcount and
 110          * locking down the physical addresses. */
 111         PMRRefPMR(psPMR);
 112
 113         if (PMRLockSysPhysAddresses(psPMR) != PVRSRV_OK)
 114         {
 115                 PVR_DPF((PVR_DBG_ERROR, "%s: Could not lock down physical addresses, aborting.", __func__));
 116                 PMRUnrefPMR(psPMR);
 117         }
 118 }
 119
 120 static void MMapPMRClose(struct vm_area_struct *ps_vma)
 121 {
 122         PMR *psPMR = ps_vma->vm_private_data;
 123
 124 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
 125 #if     defined(PVRSRV_ENABLE_MEMORY_STATS)
 126         {
 127                 uintptr_t vAddr = ps_vma->vm_start;
 128
 129                 while (vAddr < ps_vma->vm_end)
 130                 {
 131                         /* USER MAPPING */
 132                         PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, (IMG_UINT64)vAddr);
 133                         vAddr += PAGE_SIZE;
 134                 }
 135         }
 136 #else
 137         PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, ps_vma->vm_end - ps_vma->vm_start);
 138 #endif
 139 #endif
 140
 141         PMRUnlockSysPhysAddresses(psPMR);
 142         PMRUnrefPMR(psPMR);
 143 }
 144
 145 /*
 146  * This vma operation is used to read data from mmap regions. It is called
 147  * by access_process_vm, which is called to handle PTRACE_PEEKDATA ptrace
 148  * requests and reads from /proc/<pid>/mem.
 149  */
 150 static int MMapVAccess(struct vm_area_struct *ps_vma, unsigned long addr,
 151                        void *buf, int len, int write)
 152 {
 153         PMR *psPMR = ps_vma->vm_private_data;
 154         unsigned long ulOffset = addr - ps_vma->vm_start;
 155         size_t uiBytesCopied;
 156         PVRSRV_ERROR eError;
 157         int iRetVal = -EINVAL;
 158
 159         if (write)
 160         {
 161                 eError = PMR_WriteBytes(psPMR,
 162                                         (IMG_DEVMEM_OFFSET_T) ulOffset,
 163                                         buf,
 164                                         len,
 165                                         &uiBytesCopied);
 166         }
 167         else
 168         {
 169                 eError = PMR_ReadBytes(psPMR,
 170                                        (IMG_DEVMEM_OFFSET_T) ulOffset,
 171                                        buf,
 172                                        len,
 173                                        &uiBytesCopied);
 174         }
 175
 176         if (eError != PVRSRV_OK)
 177         {
 178                 PVR_DPF((PVR_DBG_ERROR, "%s: Error from %s (%d)",
 179                          __func__,
 180                          write ? "PMR_WriteBytes" : "PMR_ReadBytes",
 181                          eError));
 182         }
 183         else
 184         {
 185                 iRetVal = uiBytesCopied;
 186         }
 187
 188         return iRetVal;
 189 }
 190
 191 static const struct vm_operations_struct gsMMapOps =
 192 {
 193         .open = &MMapPMROpen,
 194         .close = &MMapPMRClose,
 195         .access = MMapVAccess,
 196 };
 197
 198 static INLINE int _OSMMapPMR(PVRSRV_DEVICE_NODE *psDevNode,
 199                                                         struct vm_area_struct *ps_vma,
 200                                                         IMG_DEVMEM_OFFSET_T uiOffset,
 201                                                         IMG_CPU_PHYADDR *psCpuPAddr,
 202                                                         IMG_UINT32 uiLog2PageSize,
 203                                                         IMG_BOOL bUseVMInsertPage,
 204                                                         IMG_BOOL bUseMixedMap)
 205 {
 206         IMG_INT32 iStatus;
 207 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 208         pfn_t sPFN;
 209 #else
 210         unsigned long uiPFN;
 211 #endif
 212
 213 #if defined(CONFIG_L4)
 214         IMG_CPU_VIRTADDR pvCpuVAddr;
 215
 216         /* Use L4LINUX function, removes per-arch code-path */
 217         pvCpuVAddr = l4x_phys_to_virt(psCpuPAddr->uiAddr);
 218         if (pvCpuVAddr == NULL)
 219         {
 220                 return -1;
 221         }
 222
 223 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 224         sPFN = phys_to_pfn_t((uintptr_t)pvCpuVAddr, 0);
 225 #else
 226         uiPFN = ((uintptr_t) pvCpuVAddr) >> PAGE_SHIFT;
 227 #endif
 228 #else /* defined(CONFIG_L4) */
 229 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 230         sPFN = phys_to_pfn_t(psCpuPAddr->uiAddr, 0);
 231 #else
 232         uiPFN = psCpuPAddr->uiAddr >> PAGE_SHIFT;
 233         PVR_ASSERT(((IMG_UINT64)uiPFN << PAGE_SHIFT) == psCpuPAddr->uiAddr);
 234 #endif
 235 #endif
 236
 237         /*
 238          * vm_insert_page() allows insertion of individual pages into user
 239          * VMA space _only_ if page is a order-zero allocated page
 240          */
 241         if (bUseVMInsertPage)
 242         {
 243                 if (bUseMixedMap)
 244                 {
 245                         /*
 246                          * This path is just for debugging. It should be
 247                          * equivalent to the remap_pfn_range() path.
 248                          */
 249                         iStatus = vm_insert_mixed(ps_vma,
 250                                                                           ps_vma->vm_start + uiOffset,
 251 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 252                                                                           sPFN);
 253 #else
 254                                                                           uiPFN);
 255 #endif
 256                 }
 257                 else
 258                 {
 259                         /* Since kernel 3.7 this sets VM_MIXEDMAP internally */
 260                         iStatus = vm_insert_page(ps_vma,
 261                                                                          ps_vma->vm_start + uiOffset,
 262 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 263                                                                          pfn_t_to_page(sPFN));
 264 #else
 265                                                                          pfn_to_page(uiPFN));
 266 #endif
 267                 }
 268         }
 269         else
 270         {
 271                 /*
 272                    NOTE: Regarding absence of dma_mmap_coherent() in _OSMMapPMR()
 273
 274                    The current services mmap model maps in a PMR's full-length size
 275                    into the user VMA & applies any user specified offset to the kernel
 276                    returned zero-offset based VA in services client; this essentially
 277                    means services server ignores ps_vma->vm_pgoff (this houses hPMR)
 278                    during a mmap call.
 279
 280                    Furthermore, during a DMA/CMA memory allocation, multiple order-n
 281                    pages are used to satisfy an allocation request due to DMA/CMA
 282                    framework rounding-up allocation size to next power-of-two which
 283                    can lead to wasted memory (so we don't allocate using single call).
 284
 285                    The combination of the above two issues mean that we cannot use the
 286                    dma_mmap_coherent() for a number of reasons outlined below:
 287
 288                      - Services mmap semantics does not fit with dma_mmap_coherent()
 289                        which requires proper ps_vma->vm_pgoff; seeing this houses a
 290                        hPMR handle value, calls into dma_mmap_coherent() fails. This
 291                        could be avoided by forcing ps_vma->vm_pgoff to zero but the
 292                        ps_vma->vm_pgoff is applied to DMA bus address PFN and not
 293                        user VMA which is always mapped at ps_vma->vm_start.
 294
 295                      - As multiple order-n pages are used for DMA/CMA allocations, a
 296                        single dma_mmap_coherent() call with a vma->vm_pgoff set to
 297                        zero cannot (maybe) be used because there is no guarantee that
 298                        all of the multiple order-n pages in the PMR are physically
 299                        contiguous from the first entry to the last. Whilst this is
 300                        highly likely to be the case, there is no guarantee that it
 301                        will be so we cannot depend on this being the case.
 302
 303                    The solution is to manually mmap DMA/CMA pages into user VMA
 304                    using remap_pfn_range() directly. Furthermore, accounting is
 305                    always compromised for DMA/CMA allocations.
 306                 */
 307                 size_t uiNumContiguousBytes = 1ULL << uiLog2PageSize;
 308
 309                 iStatus = remap_pfn_range(ps_vma,
 310                                                                   ps_vma->vm_start + uiOffset,
 311 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 312                                                                   pfn_t_to_pfn(sPFN),
 313 #else
 314                                                                   uiPFN,
 315 #endif
 316                                                                   uiNumContiguousBytes,
 317                                                                   ps_vma->vm_page_prot);
 318         }
 319
 320         return iStatus;
 321 }
 322
 323 PVRSRV_ERROR
 324 OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData)
 325 {
 326         struct vm_area_struct *ps_vma = pOSMMapData;
 327         PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
 328         PVRSRV_ERROR eError;
 329         size_t uiLength;
 330         IMG_INT32 iStatus;
 331         IMG_DEVMEM_OFFSET_T uiOffset;
 332         IMG_UINT32 ui32CPUCacheFlags;
 333         pgprot_t sPageProt;
 334         IMG_CPU_PHYADDR asCpuPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
 335         IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
 336         IMG_UINT32 uiOffsetIdx;
 337         IMG_UINT32 uiNumOfPFNs;
 338         IMG_UINT32 uiLog2PageSize;
 339         IMG_CPU_PHYADDR *psCpuPAddr;
 340         IMG_BOOL *pbValid;
 341         IMG_BOOL bUseMixedMap = IMG_FALSE;
 342         IMG_BOOL bUseVMInsertPage = IMG_FALSE;
 343
 344         eError = PMRLockSysPhysAddresses(psPMR);
 345         if (eError != PVRSRV_OK)
 346         {
 347                 goto e0;
 348         }
 349
 350         if (((ps_vma->vm_flags & VM_WRITE) != 0) &&
 351                 ((ps_vma->vm_flags & VM_SHARED) == 0))
 352         {
 353                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 354                 goto e0;
 355         }
 356
 357         sPageProt = vm_get_page_prot(ps_vma->vm_flags);
 358
 359         ui32CPUCacheFlags = DevmemCPUCacheMode(psDevNode, PMR_Flags(psPMR));
 360         switch (ui32CPUCacheFlags)
 361         {
 362                 case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
 363                                 sPageProt = pgprot_noncached(sPageProt);
 364                                 break;
 365
 366                 case PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE:
 367                                 sPageProt = pgprot_writecombine(sPageProt);
 368                                 break;
 369
 370                 case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
 371                 {
 372 /* Do not set to write-combine for plato */
 373 #if !defined(PLATO_MEMORY_CONFIG)
 374                                 PHYS_HEAP *psPhysHeap = PMR_PhysHeap(psPMR);
 375
 376                                 if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA)
 377                                         sPageProt = pgprot_writecombine(sPageProt);
 378 #endif
 379                                 break;
 380                 }
 381
 382                 default:
 383                                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 384                                 goto e0;
 385         }
 386         ps_vma->vm_page_prot = sPageProt;
 387
 388         ps_vma->vm_flags |= VM_IO;
 389
 390         /* Don't include the mapping in core dumps */
 391         ps_vma->vm_flags |= VM_DONTDUMP;
 392
 393         /*
 394          * Disable mremap because our nopage handler assumes all
 395          * page requests have already been validated.
 396          */
 397         ps_vma->vm_flags |= VM_DONTEXPAND;
 398
 399         /* Don't allow mapping to be inherited across a process fork */
 400         ps_vma->vm_flags |= VM_DONTCOPY;
 401
 402         uiLength = ps_vma->vm_end - ps_vma->vm_start;
 403
 404         /* Is this mmap targeting non order-zero pages or does it use pfn mappings?
 405          * If yes, don't use vm_insert_page */
 406         uiLog2PageSize = PMR_GetLog2Contiguity(psPMR);
 407 #if defined(PMR_OS_USE_VM_INSERT_PAGE)
 408         bUseVMInsertPage = (uiLog2PageSize == PAGE_SHIFT) && (PMR_GetType(psPMR) != PMR_TYPE_EXTMEM);
 409 #endif
 410
 411         /* Can we use stack allocations */
 412         uiNumOfPFNs = uiLength >> uiLog2PageSize;
 413         if (uiNumOfPFNs > PMR_MAX_TRANSLATION_STACK_ALLOC)
 414         {
 415                 psCpuPAddr = OSAllocMem(uiNumOfPFNs * sizeof(*psCpuPAddr));
 416                 if (psCpuPAddr == NULL)
 417                 {
 418                         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 419                         goto e1;
 420                 }
 421
 422                 /* Should allocation fail, clean-up here before exiting */
 423                 pbValid = OSAllocMem(uiNumOfPFNs * sizeof(*pbValid));
 424                 if (pbValid == NULL)
 425                 {
 426                         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 427                         OSFreeMem(psCpuPAddr);
 428                         goto e1;
 429                 }
 430         }
 431         else
 432         {
 433                 psCpuPAddr = asCpuPAddr;
 434                 pbValid = abValid;
 435         }
 436
 437         /* Obtain map range pfns */
 438         eError = PMR_CpuPhysAddr(psPMR,
 439                                  uiLog2PageSize,
 440                                  uiNumOfPFNs,
 441                                  0,
 442                                  psCpuPAddr,
 443                                  pbValid);
 444         if (eError != PVRSRV_OK)
 445         {
 446                 goto e3;
 447         }
 448
 449         /*
 450          * Scan the map range for pfns without struct page* handling. If
 451          * we find one, this is a mixed map, and we can't use vm_insert_page()
 452          * NOTE: vm_insert_page() allows insertion of individual pages into user
 453          * VMA space _only_ if said page is an order-zero allocated page.
 454          */
 455         if (bUseVMInsertPage)
 456         {
 457 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 458                 pfn_t sPFN;
 459 #else
 460                 unsigned long uiPFN;
 461 #endif
 462
 463                 for (uiOffsetIdx = 0; uiOffsetIdx < uiNumOfPFNs; ++uiOffsetIdx)
 464                 {
 465                         if (pbValid[uiOffsetIdx])
 466                         {
 467 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
 468                                 sPFN = phys_to_pfn_t(psCpuPAddr[uiOffsetIdx].uiAddr, 0);
 469
 470                                 if (!pfn_t_valid(sPFN) || page_count(pfn_t_to_page(sPFN)) == 0)
 471 #else
 472                                 uiPFN = psCpuPAddr[uiOffsetIdx].uiAddr >> PAGE_SHIFT;
 473                                 PVR_ASSERT(((IMG_UINT64)uiPFN << PAGE_SHIFT) == psCpuPAddr[uiOffsetIdx].uiAddr);
 474
 475                                 if (!pfn_valid(uiPFN) || page_count(pfn_to_page(uiPFN)) == 0)
 476 #endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)) */
 477                                 {
 478                                         bUseMixedMap = IMG_TRUE;
 479                                         break;
 480                                 }
 481                         }
 482                 }
 483
 484                 if (bUseMixedMap)
 485                 {
 486                         ps_vma->vm_flags |= VM_MIXEDMAP;
 487                 }
 488         }
 489         else
 490         {
 491                 ps_vma->vm_flags |= VM_PFNMAP;
 492         }
 493
 494         /* For each PMR page-size contiguous bytes, map page(s) into user VMA */
 495         for (uiOffset = 0; uiOffset < uiLength; uiOffset += 1ULL<<uiLog2PageSize)
 496         {
 497                 uiOffsetIdx = uiOffset >> uiLog2PageSize;
 498                 /*
 499                  * Only map in pages that are valid, any that aren't will be
 500                  * picked up by the nopage handler which will return a zeroed
 501                  * page for us.
 502                  */
 503                 if (pbValid[uiOffsetIdx])
 504                 {
 505                         iStatus = _OSMMapPMR(psDevNode,
 506                                                                  ps_vma,
 507                                                                  uiOffset,
 508                                                                  &psCpuPAddr[uiOffsetIdx],
 509                                                                  uiLog2PageSize,
 510                                                                  bUseVMInsertPage,
 511                                                                  bUseMixedMap);
 512                         if (iStatus)
 513                         {
 514                                 /* Failure error code doesn't get propagated */
 515                                 eError = PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
 516                                 PVR_ASSERT(0);
 517                                 goto e3;
 518                         }
 519                 }
 520 #if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
 521                 PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
 522                                                                         (void*)(uintptr_t)(ps_vma->vm_start + uiOffset),
 523                                                                         psCpuPAddr[uiOffsetIdx],
 524                                                                         1<<uiLog2PageSize,
 525                                                                         NULL);
 526 #endif
 527         }
 528
 529 #if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
 530         PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, uiNumOfPFNs * PAGE_SIZE);
 531 #endif
 532
 533         if (psCpuPAddr != asCpuPAddr)
 534         {
 535                 OSFreeMem(psCpuPAddr);
 536                 OSFreeMem(pbValid);
 537         }
 538
 539         /* let us see the PMR so we can unlock it later */
 540         ps_vma->vm_private_data = psPMR;
 541
 542         /* Install open and close handlers for ref-counting */
 543         ps_vma->vm_ops = &gsMMapOps;
 544
 545         /*
 546          * Take a reference on the PMR so that it can't be freed while mapped
 547          * into the user process.
 548          */
 549         PMRRefPMR(psPMR);
 550
 551         return PVRSRV_OK;
 552
 553         /* Error exit paths follow */
 554  e3:
 555         if (psCpuPAddr != asCpuPAddr)
 556         {
 557                 OSFreeMem(psCpuPAddr);
 558                 OSFreeMem(pbValid);
 559         }
 560  e1:
 561         PVR_DPF((PVR_DBG_ERROR, "don't know how to handle this error.  Abort!"));
 562         PMRUnlockSysPhysAddresses(psPMR);
 563  e0:
 564         return eError;
 565 }