/*************************************************************************/ /*!
@File
@Title          Implementation of PMR functions to wrap non-services allocated
                memory.
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@Description    Part of the memory management.  This module is responsible for
                implementing the function callbacks for physical memory.
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ /**************************************************************************/

#include <linux/version.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/mm_types.h>
#include <linux/vmalloc.h>
#include <linux/gfp.h>
#include <linux/sched.h>
#include <linux/atomic.h>
#include <asm/io.h>

#include "img_types.h"
#include "pvrsrv_error.h"
#include "pvr_debug.h"
#include "devicemem_server_utils.h"

#include "physmem_extmem.h"
#include "pdump_physmem.h"
#include "pmr.h"
#include "pmr_impl.h"

#include "kernel_compatibility.h"

typedef enum _WRAP_EXT_MEM_TYPE_ {
	WRAP_TYPE_NULL = 0,
	WRAP_TYPE_GET_USER_PAGES,
	WRAP_TYPE_FIND_VMA
} WRAP_EXT_MEM_TYPE;


typedef struct _WRAP_KERNEL_MAP_DATA_
{
	void *pvKernLinAddr;
	IMG_BOOL bVMAP;
} WRAP_KERNEL_MAP_DATA;

typedef struct _PMR_WRAP_DATA_ {

	/* Device for which this allocation has been made */
	PVRSRV_DEVICE_NODE *psDevNode;

	/* The number of physical pages or length of the
	 * page and phys address arrays below */
	IMG_UINT32 uiTotalNumPages;

	/* This is only filled if we have page mappings,
	 * for pfn mappings this stays empty */
	struct page **ppsPageArray;

	/* This should always be filled and hold the physical addresses */
	IMG_CPU_PHYADDR *ppvPhysAddr;

	/* Did we find the pages via get_user_pages()
	 * or via FindVMA and a page table walk? */
	WRAP_EXT_MEM_TYPE eWrapType;

	/* CPU caching modes for the PMR mapping */
	IMG_UINT32 ui32CPUCacheFlags;

} PMR_WRAP_DATA;


/* Free the PMR private data */
static void _FreeWrapData(PMR_WRAP_DATA *psPrivData)
{
	OSFreeMem(psPrivData->ppsPageArray);
	OSFreeMem(psPrivData->ppvPhysAddr);
	OSFreeMem(psPrivData);
}


/* Allocate the PMR private data */
static PVRSRV_ERROR _AllocWrapData(PMR_WRAP_DATA **ppsPrivData,
                            PVRSRV_DEVICE_NODE *psDevNode,
                            IMG_DEVMEM_SIZE_T uiSize,
                            PVRSRV_MEMALLOCFLAGS_T uiFlags)
{
	PVRSRV_ERROR eError;
	PMR_WRAP_DATA *psPrivData;

	/* Allocate and initialise private factory data */
	psPrivData = OSAllocZMem(sizeof(*psPrivData));
	if (psPrivData == NULL)
	{
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

	eError = DevmemCPUCacheMode(psDevNode,
	                            uiFlags,
	                            &psPrivData->ui32CPUCacheFlags);
	if (eError != PVRSRV_OK)
	{
		goto e0;
	}
	psPrivData->psDevNode = psDevNode;
	psPrivData->uiTotalNumPages = uiSize >> PAGE_SHIFT;

	/* Allocate page and phys address arrays */
	psPrivData->ppsPageArray = OSAllocZMem(sizeof(*(psPrivData->ppsPageArray)) * psPrivData->uiTotalNumPages);
	if (psPrivData == NULL)
	{
		OSFreeMem(psPrivData);
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

	psPrivData->ppvPhysAddr = OSAllocZMem(sizeof(*(psPrivData->ppvPhysAddr)) * psPrivData->uiTotalNumPages);
	if (psPrivData == NULL)
	{
		OSFreeMem(psPrivData->ppsPageArray);
		OSFreeMem(psPrivData);
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

	*ppsPrivData = psPrivData;

	return PVRSRV_OK;

e0:
	return eError;
}


/* Find the PFN associated with a given CPU virtual address, and return
 * the associated page structure, if it exists.
 * The page in question must be present (i.e. no fault handling required),
 * and must be writable. A get_page is done on the returned page structure. */
static IMG_BOOL _CPUVAddrToPFN(struct vm_area_struct *psVMArea,
                              uintptr_t uCPUVAddr,
                              unsigned long *pui32PFN,
                              struct page **ppsPage)
{
	pgd_t *psPGD;
	p4d_t *psP4D;
	pud_t *psPUD;
	pmd_t *psPMD;
	pte_t *psPTE;
	struct mm_struct *psMM = psVMArea->vm_mm;
	spinlock_t *psPTLock;
	IMG_BOOL bRet = IMG_FALSE;

	*pui32PFN = 0;
	*ppsPage = NULL;

	/* Walk the page tables to find the PTE */
	psPGD = pgd_offset(psMM, uCPUVAddr);
	if (pgd_none(*psPGD) || pgd_bad(*psPGD))
		return bRet;

	psP4D = p4d_offset(psPGD, uCPUVAddr);
	if (p4d_none(*psP4D) || unlikely(p4d_bad(*psP4D)))
		return bRet;

	psPUD = pud_offset(psP4D, uCPUVAddr);
	if (pud_none(*psPUD) || pud_bad(*psPUD))
		return bRet;

	psPMD = pmd_offset(psPUD, uCPUVAddr);
	if (pmd_none(*psPMD) || pmd_bad(*psPMD))
		return bRet;

	psPTE = (pte_t *)pte_offset_map_lock(psMM, psPMD, uCPUVAddr, &psPTLock);

	/* Check if the returned PTE is actually valid and writable */
	if ((pte_none(*psPTE) == 0) && (pte_present(*psPTE) != 0) && (pte_write(*psPTE) != 0))
	{
		*pui32PFN = pte_pfn(*psPTE);
		bRet = IMG_TRUE;

		/* In case the pfn is valid, meaning it is a RAM page and not
		 * IO-remapped, we can get the actual page struct from it. */
		if (pfn_valid(*pui32PFN))
		{
			*ppsPage = pfn_to_page(*pui32PFN);

			get_page(*ppsPage);
		}
	}

	pte_unmap_unlock(psPTE, psPTLock);

	return bRet;
}

/* Free the pages we got via _TryFindVMA() */
static void _FreeFindVMAPages(PMR_WRAP_DATA *psPrivData)
{
	IMG_UINT32 i;

	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
	{
		if (psPrivData->ppsPageArray[i] != NULL)
		{
			/* Release the page */
			put_page(psPrivData->ppsPageArray[i]);
		}
	}
}

/* Find the VMA to a given CPU virtual address and do a
 * page table walk to find the corresponding pfns */
static IMG_BOOL _TryFindVMA(IMG_DEVMEM_SIZE_T uiSize,
                            uintptr_t pvCpuVAddr,
                            PMR_WRAP_DATA *psPrivData)
{
	struct vm_area_struct *psVMArea;
	uintptr_t pvCpuVAddrEnd = pvCpuVAddr + uiSize;
	IMG_UINT32 i;
	uintptr_t uAddr;

	down_read(&current->mm->mmap_sem);

	/* Find the VMA */
	psVMArea = find_vma(current->mm, pvCpuVAddr);
	if (psVMArea == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Couldn't find memory region containing start address %p",
				__func__,
				(void*) pvCpuVAddr));
		goto e0;
	}

	/* Find_vma locates a region with an end point past a given
	 * virtual address. So check the address is actually in the region. */
	if (pvCpuVAddr < psVMArea->vm_start)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Start address %p is outside of the region returned by find_vma",
				__func__,
				(void*) pvCpuVAddr));
		goto e0;
	}

	/* Now check the end address is in range */
	if (pvCpuVAddrEnd > psVMArea->vm_end)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: End address %p is outside of the region returned by find_vma",
				__func__,
				(void*) pvCpuVAddrEnd));
		goto e0;
	}

	/* Does the region represent memory mapped I/O? */
	if (!(psVMArea->vm_flags & VM_IO))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Memory region does not represent memory mapped I/O (VMA flags: 0x%lx)",
				__func__,
				psVMArea->vm_flags));
		goto e0;
	}

	/* We require read and write access */
	if ((psVMArea->vm_flags & (VM_READ | VM_WRITE)) != (VM_READ | VM_WRITE))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: No read/write access to memory region (VMA flags: 0x%lx)",
				__func__,
				psVMArea->vm_flags));
		goto e0;
	}

	/* Do the actual page table walk and fill the private data arrays
	 * for page structs and physical addresses */
	for (uAddr = pvCpuVAddr, i = 0;
	     uAddr < pvCpuVAddrEnd;
	     uAddr += PAGE_SIZE, i++)
	{
		unsigned long ui32PFN = 0;

		PVR_ASSERT(i < psPrivData->uiTotalNumPages);

		if ( !_CPUVAddrToPFN(psVMArea, uAddr, &ui32PFN, &psPrivData->ppsPageArray[i]) )
		{
			PVR_DPF((PVR_DBG_ERROR,
					"%s: Invalid CPU virtual address",
					__func__));
			goto e1;
		}

		psPrivData->ppvPhysAddr[i].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(ui32PFN << PAGE_SHIFT);

		if ( (((IMG_UINT32) psPrivData->ppvPhysAddr[i].uiAddr) >> PAGE_SHIFT) != ui32PFN)
		{
			PVR_DPF((PVR_DBG_ERROR,
					"%s: Page frame number out of range (%lu)",
					__func__,
					ui32PFN));
			goto e1;
		}
	}

	/* Sanity check */
	PVR_ASSERT(i == psPrivData->uiTotalNumPages);

	up_read(&current->mm->mmap_sem);

	return IMG_TRUE;

e1:
	for (; i != 0; i--)
	{
		if (psPrivData->ppsPageArray[i-1] != NULL)
		{
			put_page(psPrivData->ppsPageArray[i-1]);
		}
	}
e0:
	up_read(&current->mm->mmap_sem);
	return IMG_FALSE;
}

/* Release pages got via get_user_pages().
 * As the function comment for GUP states we have
 * to set the page dirty if it has been written to and
 * we have to put the page to decrease its refcount. */
static void _FreeGetUserPages(PMR_WRAP_DATA *psPrivData)
{
	IMG_UINT32 i;

	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
	{
		/* Write data back to fs if necessary */
		set_page_dirty_lock(psPrivData->ppsPageArray[i]);

		/* Release the page */
		put_page(psPrivData->ppsPageArray[i]);
	}
}

/* Get the page structures and physical addresses mapped to
 * a CPU virtual range via get_user_pages() */
static IMG_BOOL _TryGetUserPages(IMG_DEVMEM_SIZE_T uiSize,
                                IMG_CPU_VIRTADDR pvCpuVAddr,
                                PMR_WRAP_DATA *psPrivData)
{
	IMG_INT32 iMappedPages, i;

	down_read(&current->mm->mmap_sem);

	/* Do the actual call */
	iMappedPages = get_user_pages((uintptr_t) pvCpuVAddr,
	                              uiSize >> PAGE_SHIFT,
	                              FOLL_WRITE,
	                              psPrivData->ppsPageArray,
	                              NULL);
	if (iMappedPages == 0 || iMappedPages != psPrivData->uiTotalNumPages)
	{
		up_read(&current->mm->mmap_sem);

		PVR_DPF((PVR_DBG_MESSAGE,
		         "get_user_pages() failed, got back %d, expected num pages %d",
		         iMappedPages,
		         psPrivData->uiTotalNumPages));

		return IMG_FALSE;
	}

	/* Fill the physical address array */
	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
	{
		psPrivData->ppvPhysAddr[i].uiAddr = page_to_phys(psPrivData->ppsPageArray[i]);
	}

	up_read(&current->mm->mmap_sem);

	return IMG_TRUE;
}

/* Release all the pages we got via _TryGetUserPages or _TryFindVMA */
static PVRSRV_ERROR _WrapExtMemReleasePages(PMR_WRAP_DATA *psPrivData)
{
	switch (psPrivData->eWrapType)
	{
		case WRAP_TYPE_GET_USER_PAGES:
			_FreeGetUserPages(psPrivData);
			break;
		case WRAP_TYPE_FIND_VMA:
			_FreeFindVMAPages(psPrivData);
			break;
		case WRAP_TYPE_NULL:
			/* fall through */
		default:
			PVR_DPF((PVR_DBG_ERROR,
					"%s: Wrong wrap type, cannot release pages",
					__func__));
			return PVRSRV_ERROR_INVALID_WRAP_TYPE;
	}

	_FreeWrapData(psPrivData);

	return PVRSRV_OK;
}

/* Try to get pages and physical addresses for a CPU virtual address.
 * Will try both methods:
 * get_user_pages or find_vma + page table walk if the first one fails */
static PVRSRV_ERROR _WrapExtMemAcquirePages(IMG_DEVMEM_SIZE_T uiSize,
                                            IMG_CPU_VIRTADDR pvCpuVAddr,
                                            PMR_WRAP_DATA *psPrivData)
{
	IMG_BOOL bSuccess;

	bSuccess = _TryGetUserPages(uiSize,
	                            pvCpuVAddr,
	                            psPrivData);
	if (bSuccess)
	{
		psPrivData->eWrapType = WRAP_TYPE_GET_USER_PAGES;
		PVR_DPF((PVR_DBG_MESSAGE,
				"%s: Used GetUserPages",
				__func__));
		return PVRSRV_OK;
	}

	bSuccess = _TryFindVMA(uiSize,
	                       (uintptr_t) pvCpuVAddr,
	                       psPrivData);
	if (bSuccess)
	{
		psPrivData->eWrapType = WRAP_TYPE_FIND_VMA;
		PVR_DPF((PVR_DBG_MESSAGE,
				"%s: Used FindVMA",
				__func__));
		return PVRSRV_OK;
	}

	psPrivData->eWrapType = WRAP_TYPE_NULL;

	PVR_DPF((PVR_DBG_ERROR,
			"%s: Unable to find all physical pages related "
			"to this CPU virtual address and size.",
			__func__));

	return PVRSRV_ERROR_INVALID_CPU_ADDR;
}

static PVRSRV_ERROR
PMRSysPhysAddrExtMem(PMR_IMPL_PRIVDATA pvPriv,
                     IMG_UINT32 ui32Log2PageSize,
                     IMG_UINT32 ui32NumOfPages,
                     IMG_DEVMEM_OFFSET_T *puiOffset,
                     IMG_BOOL *pbValid,
                     IMG_DEV_PHYADDR *psDevPAddr)
{
	const PMR_WRAP_DATA *psWrapData = pvPriv;
	IMG_UINT32 uiPageSize = 1U << PAGE_SHIFT;
	IMG_UINT32 uiInPageOffset;
	IMG_UINT32 uiPageIndex;
	IMG_UINT32 uiIdx;

	PVR_UNREFERENCED_PARAMETER(pbValid);

	if (PAGE_SHIFT != ui32Log2PageSize)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Requested ui32Log2PageSize %u is different from "
				"OS page shift %u. Not supported.",
				__func__,
				ui32Log2PageSize,
				PAGE_SHIFT));
		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
	}

	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
	{
		uiPageIndex = puiOffset[uiIdx] >> PAGE_SHIFT;
		uiInPageOffset = puiOffset[uiIdx] - ((IMG_DEVMEM_OFFSET_T)uiPageIndex << PAGE_SHIFT);

		PVR_ASSERT(uiPageIndex < psWrapData->uiTotalNumPages);
		PVR_ASSERT(uiInPageOffset < uiPageSize);

		/* We always handle CPU physical addresses in this PMR factory
		 * but this callback expects device physical addresses so we have to translate. */
		PhysHeapCpuPAddrToDevPAddr(psWrapData->psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_CPU_LOCAL],
		                           1,
		                           &psDevPAddr[uiIdx],
		                           &psWrapData->ppvPhysAddr[uiPageIndex]);


		psDevPAddr[uiIdx].uiAddr += uiInPageOffset;
	}

	return PVRSRV_OK;
}


static PVRSRV_ERROR
PMRFinalizeExtMem(PMR_IMPL_PRIVDATA pvPriv)
{
	PVRSRV_ERROR eError;
	PMR_WRAP_DATA *psWrapData = pvPriv;

	eError = _WrapExtMemReleasePages(psWrapData);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Problem with _WrapExtMemReleasePages. "
				"Not all resources could be cleaned up (%u).",
				__func__,
				eError));
	}

	return PVRSRV_OK;
}


static void _UnmapPage(PMR_WRAP_DATA *psWrapData,
                       WRAP_KERNEL_MAP_DATA *psMapData)
{
	IMG_BOOL bSuccess;

	if (psMapData->bVMAP)
	{
		vunmap(psMapData->pvKernLinAddr);
	}
	else
	{
		bSuccess = OSUnMapPhysToLin(psMapData->pvKernLinAddr,
		                            PAGE_SIZE,
		                            psWrapData->ui32CPUCacheFlags);
		if (!bSuccess)
		{
			PVR_DPF((PVR_DBG_ERROR,
					"%s: Unable to unmap wrapped extmem page. "
					"This may leak memory.",
					__func__));
		}
	}
}

static void _MapPage(PMR_WRAP_DATA *psWrapData,
                     IMG_UINT32 uiPageIdx,
                     WRAP_KERNEL_MAP_DATA *psMapData)
{
	IMG_UINT8 *puiLinCPUAddr;

	if (psWrapData->ppsPageArray[uiPageIdx])
	{
		pgprot_t prot = PAGE_KERNEL;

		switch (PVRSRV_CPU_CACHE_MODE(psWrapData->ui32CPUCacheFlags))
		{
			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
				prot = pgprot_noncached(prot);
				break;

			case PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE:
				prot = pgprot_writecombine(prot);
				break;

			case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
				break;

			default:
				break;
		}

		puiLinCPUAddr = vmap(&psWrapData->ppsPageArray[uiPageIdx],
		                       1,
		                       VM_READ | VM_WRITE,
		                       prot);

		psMapData->bVMAP = IMG_TRUE;
	}
	else
	{
		puiLinCPUAddr = (IMG_UINT8*) OSMapPhysToLin(psWrapData->ppvPhysAddr[uiPageIdx],
		                                              PAGE_SIZE,
		                                              psWrapData->ui32CPUCacheFlags);

		psMapData->bVMAP = IMG_FALSE;
	}

	psMapData->pvKernLinAddr = puiLinCPUAddr;

}

static PVRSRV_ERROR _CopyBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
                                     IMG_DEVMEM_OFFSET_T uiOffset,
                                     IMG_UINT8 *pcBuffer,
                                     size_t uiBufSz,
                                     size_t *puiNumBytes,
                                     IMG_BOOL bWrite)
{
	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
	size_t uiBytesToCopy = uiBufSz;
	IMG_UINT32 uiPageIdx =  uiOffset >> PAGE_SHIFT;
	IMG_BOOL bFirst = IMG_TRUE;
	WRAP_KERNEL_MAP_DATA sKernMapData;


	if ((uiBufSz + uiOffset) > (psWrapData->uiTotalNumPages << PAGE_SHIFT))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Trying to read out of bounds of PMR.",
				__func__));
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	/* Copy the pages */
	while (uiBytesToCopy != 0)
	{
		size_t uiBytesPerLoop;
		IMG_DEVMEM_OFFSET_T uiCopyOffset;

		if (bFirst)
		{
			bFirst = IMG_FALSE;
			uiCopyOffset = (uiOffset & ~PAGE_MASK);
			uiBytesPerLoop = (uiBufSz >= (PAGE_SIZE - uiCopyOffset)) ?
					PAGE_SIZE - uiCopyOffset : uiBufSz;
		}
		else
		{
			uiCopyOffset = 0;
			uiBytesPerLoop = (uiBytesToCopy > PAGE_SIZE) ? PAGE_SIZE : uiBytesToCopy;
		}

		_MapPage(psWrapData,
		         uiPageIdx,
		         &sKernMapData);

		if (sKernMapData.pvKernLinAddr == NULL)
		{
			PVR_DPF((PVR_DBG_ERROR,
					"%s: Unable to map wrapped extmem page.",
					__func__));
			return PVRSRV_ERROR_OUT_OF_MEMORY;
		}

		/* Use 'DeviceMemCopy' because we need to be conservative. We can't
		 * know whether the wrapped memory was originally imported as cached.
		 */

		if (bWrite)
		{
			OSDeviceMemCopy(sKernMapData.pvKernLinAddr + uiCopyOffset,
			          pcBuffer,
			          uiBytesPerLoop);
		}
		else
		{
			OSDeviceMemCopy(pcBuffer,
			          sKernMapData.pvKernLinAddr + uiCopyOffset,
			          uiBytesPerLoop);
		}

		_UnmapPage(psWrapData,
		           &sKernMapData);

		uiBytesToCopy -= uiBytesPerLoop;
		uiPageIdx++;
	}

	*puiNumBytes = uiBufSz;

	return PVRSRV_OK;
}

static PVRSRV_ERROR
PMRReadBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
                   IMG_DEVMEM_OFFSET_T uiOffset,
                   IMG_UINT8 *pcBuffer,
                   size_t uiBufSz,
                   size_t *puiNumBytes)
{

	return _CopyBytesExtMem(pvPriv,
	                        uiOffset,
	                        pcBuffer,
	                        uiBufSz,
	                        puiNumBytes,
	                        IMG_FALSE);
}

static PVRSRV_ERROR
PMRWriteBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
                    IMG_DEVMEM_OFFSET_T uiOffset,
                    IMG_UINT8 *pcBuffer,
                    size_t uiBufSz,
                    size_t *puiNumBytes)
{
	return _CopyBytesExtMem(pvPriv,
	                        uiOffset,
	                        pcBuffer,
	                        uiBufSz,
	                        puiNumBytes,
	                        IMG_TRUE);
}


static PVRSRV_ERROR
PMRAcquireKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
                                  size_t uiOffset,
                                  size_t uiSize,
                                  void **ppvKernelAddressOut,
                                  IMG_HANDLE *phHandleOut,
                                  PMR_FLAGS_T ulFlags)
{
	PVRSRV_ERROR eError;
	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
	WRAP_KERNEL_MAP_DATA *psKernMapData;
	IMG_UINT32 ui32PageIndex = uiOffset >> PAGE_SHIFT;

	/* Offset was out of bounds */
	if (ui32PageIndex > psWrapData->uiTotalNumPages)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Error, offset out of PMR bounds.",
				__func__));
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e0;
	}

	/* We can not map in more than one page with ioremap.
	 * Only possible with physically contiguous pages */
	if (((uiOffset & PAGE_MASK) + uiSize) > PAGE_SIZE)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Error, cannot map more than one page for wrapped extmem.",
				__func__));
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e0;
	}

	psKernMapData = OSAllocMem(sizeof(*psKernMapData));
	if (psKernMapData == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Error, unable to allocate memory for kernel mapping data.",
				__func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

	_MapPage(psWrapData,
	         ui32PageIndex,
	         psKernMapData);

	if (psKernMapData->pvKernLinAddr == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Unable to map wrapped extmem page.",
				__func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e1;
	}

	*ppvKernelAddressOut = ((IMG_CHAR *) psKernMapData->pvKernLinAddr) + (uiOffset & PAGE_MASK);
	*phHandleOut = psKernMapData;

	return PVRSRV_OK;

	/* error exit paths follow */
e1:
	OSFreeMem(psKernMapData);
e0:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}

static void PMRReleaseKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
                                              IMG_HANDLE hHandle)
{
	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
	WRAP_KERNEL_MAP_DATA *psKernMapData = (void*) hHandle;

	_UnmapPage(psWrapData,
	           psKernMapData);

	OSFreeMem(psKernMapData);
}

static PMR_IMPL_FUNCTAB _sPMRWrapPFuncTab = {
    .pfnLockPhysAddresses = NULL,
    .pfnUnlockPhysAddresses = NULL,
    .pfnDevPhysAddr = &PMRSysPhysAddrExtMem,
    .pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataExtMem,
    .pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataExtMem,
    .pfnReadBytes = PMRReadBytesExtMem,
    .pfnWriteBytes = PMRWriteBytesExtMem,
    .pfnUnpinMem = NULL,
    .pfnPinMem = NULL,
    .pfnChangeSparseMem = NULL,
    .pfnChangeSparseMemCPUMap = NULL,
    .pfnFinalize = &PMRFinalizeExtMem,
};

PVRSRV_ERROR
PhysmemWrapExtMemOS(CONNECTION_DATA * psConnection,
                    PVRSRV_DEVICE_NODE *psDevNode,
                    IMG_DEVMEM_SIZE_T uiSize,
                    IMG_CPU_VIRTADDR pvCpuVAddr,
                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
                    PMR **ppsPMRPtr)
{
	PVRSRV_ERROR eError;
	IMG_BOOL bMappingTable = IMG_TRUE;
	PMR_WRAP_DATA *psPrivData;
	PMR *psPMR;

	/* Fail if requesting coherency on one side but uncached on the other */
	if ( (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
	         (PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags))) )
	{
		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
				"Please use GPU cached flags for coherency."));
		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
	}

	if ( (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
	         (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))) )
	{
		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
				"Please use CPU cached flags for coherency."));
		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
	}

	/* Check address and size alignment */
	if (uiSize & (PAGE_SHIFT-1))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Given size %llu is not multiple of OS page size (%lu)",
				__func__,
				uiSize,
				PAGE_SIZE));
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e0;
	}

	if ( ((uintptr_t) pvCpuVAddr) & (PAGE_SHIFT-1) )
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Given address %p is not aligned to OS page size (%lu)",
				__func__,
				pvCpuVAddr,
				PAGE_SIZE));
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e0;
	}

	/* Allocate private factory data */
	eError = _AllocWrapData(&psPrivData,
	                        psDevNode,
	                        uiSize,
	                        uiFlags);
	if (eError != PVRSRV_OK)
	{
		goto e0;
	}

	/* Actually find and acquire the pages and physical addresses */
	eError = _WrapExtMemAcquirePages(uiSize,
	                                 pvCpuVAddr,
	                                 psPrivData);
	if (eError != PVRSRV_OK)
	{
		goto e1;
	}

	/* Create a suitable PMR */
	eError = PMRCreatePMR(psDevNode,
	                      psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_CPU_LOCAL],
	                      uiSize,               /* PMR_SIZE_T uiLogicalSize                  */
	                      uiSize,               /* PMR_SIZE_T uiChunkSize                    */
	                      1,                    /* IMG_UINT32 ui32NumPhysChunks              */
	                      1,                    /* IMG_UINT32 ui32NumVirtChunks              */
	                      &bMappingTable,       /* IMG_BOOL bMappingTable                    */
	                      PAGE_SHIFT,           /* PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee */
	                      (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK), /* PMR_FLAGS_T uiFlags */
	                      "WrappedExtMem",      /* const IMG_CHAR *pszAnnotation             */
	                      &_sPMRWrapPFuncTab,   /* const PMR_IMPL_FUNCTAB *psFuncTab         */
	                      psPrivData,           /* PMR_IMPL_PRIVDATA pvPrivData              */
	                      PMR_TYPE_EXTMEM,
	                      &psPMR,               /* PMR **ppsPMRPtr                           */
	                      PDUMP_NONE);          /* IMG_UINT32 ui32PDumpFlags                 */
	if (eError != PVRSRV_OK)
	{
		goto e2;
	}

	*ppsPMRPtr = psPMR;

	return PVRSRV_OK;

e2:
	_WrapExtMemReleasePages(psPrivData);
e1:
	_FreeWrapData(psPrivData);
e0:
	return eError;

}

