static int kgsl_iommu_close(struct kgsl_mmu *mmu)
{
struct kgsl_iommu *iommu = mmu->priv;
int i;
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_pagetable *pagetable = (mmu->priv_bank_table ?
mmu->priv_bank_table : mmu->defaultpagetable);
if (iommu->iommu_units[i].reg_map.gpuaddr)
kgsl_mmu_unmap(pagetable,
&(iommu->iommu_units[i].reg_map));
if (iommu->iommu_units[i].reg_map.hostptr)
iounmap(iommu->iommu_units[i].reg_map.hostptr);
kgsl_sg_free(iommu->iommu_units[i].reg_map.sg,
iommu->iommu_units[i].reg_map.sglen);
}
if (mmu->priv_bank_table)
kgsl_mmu_putpagetable(mmu->priv_bank_table);
if (mmu->defaultpagetable)
kgsl_mmu_putpagetable(mmu->defaultpagetable);
kfree(iommu->asids);
kfree(iommu);
return 0;
}
/*
* kgsl_iommu_setup_defaultpagetable - Setup the initial defualtpagetable
* for iommu. This function is only called once during first start, successive
* start do not call this funciton.
* @mmu - Pointer to mmu structure
*
* Create the initial defaultpagetable and setup the iommu mappings to it
* Return - 0 on success else error code
*/
static int kgsl_iommu_setup_defaultpagetable(struct kgsl_mmu *mmu)
{
int status = 0;
int i = 0;
struct kgsl_iommu *iommu = mmu->priv;
struct kgsl_pagetable *pagetable = NULL;
/* If chip is not 8960 then we use the 2nd context bank for pagetable
* switching on the 3D side for which a separate table is allocated */
if (!cpu_is_msm8960() && msm_soc_version_supports_iommu_v1()) {
mmu->priv_bank_table =
kgsl_mmu_getpagetable(KGSL_MMU_PRIV_BANK_TABLE_NAME);
if (mmu->priv_bank_table == NULL) {
status = -ENOMEM;
goto err;
}
}
mmu->defaultpagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
/* Return error if the default pagetable doesn't exist */
if (mmu->defaultpagetable == NULL) {
status = -ENOMEM;
goto err;
}
pagetable = mmu->priv_bank_table ? mmu->priv_bank_table :
mmu->defaultpagetable;
/* Map the IOMMU regsiters to only defaultpagetable */
if (msm_soc_version_supports_iommu_v1()) {
for (i = 0; i < iommu->unit_count; i++) {
iommu->iommu_units[i].reg_map.priv |=
KGSL_MEMFLAGS_GLOBAL;
status = kgsl_mmu_map(pagetable,
&(iommu->iommu_units[i].reg_map),
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (status) {
iommu->iommu_units[i].reg_map.priv &=
~KGSL_MEMFLAGS_GLOBAL;
goto err;
}
}
}
return status;
err:
for (i--; i >= 0; i--) {
kgsl_mmu_unmap(pagetable,
&(iommu->iommu_units[i].reg_map));
iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL;
}
if (mmu->priv_bank_table) {
kgsl_mmu_putpagetable(mmu->priv_bank_table);
mmu->priv_bank_table = NULL;
}
if (mmu->defaultpagetable) {
kgsl_mmu_putpagetable(mmu->defaultpagetable);
mmu->defaultpagetable = NULL;
}
return status;
}
static int kgsl_iommu_setup_defaultpagetable(struct kgsl_mmu *mmu)
{
int status = 0;
int i = 0;
struct kgsl_iommu *iommu = mmu->priv;
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_pagetable *pagetable = NULL;
if (!cpu_is_msm8960()) {
mmu->priv_bank_table =
kgsl_mmu_getpagetable(KGSL_MMU_PRIV_BANK_TABLE_NAME);
if (mmu->priv_bank_table == NULL) {
status = -ENOMEM;
goto err;
}
iommu_pt = mmu->priv_bank_table->priv;
}
mmu->defaultpagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
if (mmu->defaultpagetable == NULL) {
status = -ENOMEM;
goto err;
}
pagetable = mmu->priv_bank_table ? mmu->priv_bank_table :
mmu->defaultpagetable;
for (i = 0; i < iommu->unit_count; i++) {
iommu->iommu_units[i].reg_map.priv |= KGSL_MEMFLAGS_GLOBAL;
status = kgsl_mmu_map(pagetable,
&(iommu->iommu_units[i].reg_map),
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (status) {
iommu->iommu_units[i].reg_map.priv &=
~KGSL_MEMFLAGS_GLOBAL;
goto err;
}
}
return status;
err:
for (i--; i >= 0; i--) {
kgsl_mmu_unmap(pagetable,
&(iommu->iommu_units[i].reg_map));
iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL;
}
if (mmu->priv_bank_table) {
kgsl_mmu_putpagetable(mmu->priv_bank_table);
mmu->priv_bank_table = NULL;
}
if (mmu->defaultpagetable) {
kgsl_mmu_putpagetable(mmu->defaultpagetable);
mmu->defaultpagetable = NULL;
}
return status;
}
/*
* kgsl_iommu_setup_defaultpagetable - Setup the initial defualtpagetable
* for iommu. This function is only called once during first start, successive
* start do not call this funciton.
* @mmu - Pointer to mmu structure
*
* Create the initial defaultpagetable and setup the iommu mappings to it
* Return - 0 on success else error code
*/
static int kgsl_iommu_setup_defaultpagetable(struct kgsl_mmu *mmu)
{
int status = 0;
int i = 0;
struct kgsl_iommu *iommu = mmu->priv;
struct kgsl_iommu_pt *iommu_pt;
mmu->defaultpagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
/* Return error if the default pagetable doesn't exist */
if (mmu->defaultpagetable == NULL) {
status = -ENOMEM;
goto err;
}
/* Map the IOMMU regsiters to only defaultpagetable */
for (i = 0; i < iommu->unit_count; i++) {
iommu->iommu_units[i].reg_map.priv |= KGSL_MEMFLAGS_GLOBAL;
status = kgsl_mmu_map(mmu->defaultpagetable,
&(iommu->iommu_units[i].reg_map),
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (status) {
iommu->iommu_units[i].reg_map.priv &=
~KGSL_MEMFLAGS_GLOBAL;
goto err;
}
}
/*
* The dafault pagetable always has asid 0 assigned by the iommu driver
* and asid 1 is assigned to the private context.
*/
iommu_pt = mmu->defaultpagetable->priv;
iommu_pt->asid = 0;
set_bit(0, iommu->asids);
set_bit(1, iommu->asids);
return status;
err:
for (i--; i >= 0; i--) {
kgsl_mmu_unmap(mmu->defaultpagetable,
&(iommu->iommu_units[i].reg_map));
iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL;
}
if (mmu->defaultpagetable) {
kgsl_mmu_putpagetable(mmu->defaultpagetable);
mmu->defaultpagetable = NULL;
}
return status;
}
int
kgsl_sharedmem_free0(gsl_memdesc_t *memdesc, unsigned int pid)
{
int status = GSL_SUCCESS;
int aperture_index;
gsl_deviceid_t device_id;
gsl_sharedmem_t *shmem;
kgsl_log_write( KGSL_LOG_GROUP_MEMORY | KGSL_LOG_LEVEL_TRACE, "--> int kgsl_sharedmem_free(gsl_memdesc_t *memdesc=%M)\n", memdesc );
GSL_MEMDESC_APERTURE_GET(memdesc, aperture_index);
GSL_MEMDESC_DEVICE_GET(memdesc, device_id);
shmem = &gsl_driver.shmem;
if (shmem->flags & GSL_FLAGS_INITIALIZED)
{
if (kgsl_memarena_isvirtualized(shmem->apertures[aperture_index].memarena))
{
status |= kgsl_mmu_unmap(&gsl_driver.device[device_id-1].mmu, memdesc->gpuaddr, memdesc->size, pid);
if (!GSL_MEMDESC_EXTALLOC_ISMARKED(memdesc))
{
status |= kgsl_hal_freephysical(memdesc->gpuaddr, memdesc->size / GSL_PAGESIZE, NULL);
}
}
kgsl_memarena_free(shmem->apertures[aperture_index].memarena, memdesc);
// clear descriptor
memset(memdesc, 0, sizeof(gsl_memdesc_t));
}
else
{
status = GSL_FAILURE;
}
kgsl_log_write( KGSL_LOG_GROUP_MEMORY | KGSL_LOG_LEVEL_TRACE, "<-- kgsl_sharedmem_free. Return value %B\n", status );
return (status);
}
/*
* kgsl_get_sync_lock - Init Sync Lock between GPU and CPU
* @mmu - Pointer to mmu device
*
* Return - 0 on success else error code
*/
static int kgsl_iommu_init_sync_lock(struct kgsl_mmu *mmu)
{
struct kgsl_iommu *iommu = mmu->device->mmu.priv;
int status = 0;
struct kgsl_pagetable *pagetable = NULL;
uint32_t lock_gpu_addr = 0;
uint32_t lock_phy_addr = 0;
uint32_t page_offset = 0;
iommu->sync_lock_initialized = 0;
if (!(mmu->flags & KGSL_MMU_FLAGS_IOMMU_SYNC)) {
KGSL_DRV_ERR(mmu->device,
"The GPU microcode does not support IOMMUv1 sync opcodes\n");
return -ENXIO;
}
/* Get the physical address of the Lock variables */
lock_phy_addr = (msm_iommu_lock_initialize()
- MSM_SHARED_RAM_BASE + msm_shared_ram_phys);
if (!lock_phy_addr) {
KGSL_DRV_ERR(mmu->device,
"GPU CPU sync lock is not supported by kernel\n");
return -ENXIO;
}
/* Align the physical address to PAGE boundary and store the offset */
page_offset = (lock_phy_addr & (PAGE_SIZE - 1));
lock_phy_addr = (lock_phy_addr & ~(PAGE_SIZE - 1));
iommu->sync_lock_desc.physaddr = (unsigned int)lock_phy_addr;
iommu->sync_lock_desc.size =
PAGE_ALIGN(sizeof(kgsl_iommu_sync_lock_vars));
status = memdesc_sg_phys(&iommu->sync_lock_desc,
iommu->sync_lock_desc.physaddr,
iommu->sync_lock_desc.size);
if (status)
return status;
/* Map Lock variables to GPU pagetable */
iommu->sync_lock_desc.priv |= KGSL_MEMFLAGS_GLOBAL;
pagetable = mmu->priv_bank_table ? mmu->priv_bank_table :
mmu->defaultpagetable;
status = kgsl_mmu_map(pagetable, &iommu->sync_lock_desc,
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (status) {
kgsl_mmu_unmap(pagetable, &iommu->sync_lock_desc);
iommu->sync_lock_desc.priv &= ~KGSL_MEMFLAGS_GLOBAL;
return status;
}
/* Store Lock variables GPU address */
lock_gpu_addr = (iommu->sync_lock_desc.gpuaddr + page_offset);
kgsl_iommu_sync_lock_vars.flag[PROC_APPS] = (lock_gpu_addr +
(offsetof(struct remote_iommu_petersons_spinlock,
flag[PROC_APPS])));
kgsl_iommu_sync_lock_vars.flag[PROC_GPU] = (lock_gpu_addr +
(offsetof(struct remote_iommu_petersons_spinlock,
flag[PROC_GPU])));
kgsl_iommu_sync_lock_vars.turn = (lock_gpu_addr +
(offsetof(struct remote_iommu_petersons_spinlock, turn)));
iommu->sync_lock_vars = &kgsl_iommu_sync_lock_vars;
/* Flag Sync Lock is Initialized */
iommu->sync_lock_initialized = 1;
return status;
}
/*
* kgsl_iommu_setup_defaultpagetable - Setup the initial defualtpagetable
* for iommu. This function is only called once during first start, successive
* start do not call this funciton.
* @mmu - Pointer to mmu structure
*
* Create the initial defaultpagetable and setup the iommu mappings to it
* Return - 0 on success else error code
*/
static int kgsl_iommu_setup_defaultpagetable(struct kgsl_mmu *mmu)
{
int status = 0;
int i = 0;
struct kgsl_iommu *iommu = mmu->priv;
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_pagetable *pagetable = NULL;
/* If chip is not 8960 then we use the 2nd context bank for pagetable
* switching on the 3D side for which a separate table is allocated */
if (!cpu_is_msm8960()) {
mmu->priv_bank_table =
kgsl_mmu_getpagetable(KGSL_MMU_PRIV_BANK_TABLE_NAME);
if (mmu->priv_bank_table == NULL) {
status = -ENOMEM;
goto err;
}
iommu_pt = mmu->priv_bank_table->priv;
iommu_pt->asid = 1;
}
mmu->defaultpagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
/* Return error if the default pagetable doesn't exist */
if (mmu->defaultpagetable == NULL) {
status = -ENOMEM;
goto err;
}
pagetable = mmu->priv_bank_table ? mmu->priv_bank_table :
mmu->defaultpagetable;
/* Map the IOMMU regsiters to only defaultpagetable */
for (i = 0; i < iommu->unit_count; i++) {
iommu->iommu_units[i].reg_map.priv |= KGSL_MEMFLAGS_GLOBAL;
status = kgsl_mmu_map(pagetable,
&(iommu->iommu_units[i].reg_map),
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (status) {
iommu->iommu_units[i].reg_map.priv &=
~KGSL_MEMFLAGS_GLOBAL;
goto err;
}
}
/*
* The dafault pagetable always has asid 0 assigned by the iommu driver
* and asid 1 is assigned to the private context.
*/
iommu_pt = mmu->defaultpagetable->priv;
iommu_pt->asid = 0;
set_bit(0, iommu->asids);
set_bit(1, iommu->asids);
return status;
err:
for (i--; i >= 0; i--) {
kgsl_mmu_unmap(pagetable,
&(iommu->iommu_units[i].reg_map));
iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL;
}
if (mmu->priv_bank_table) {
kgsl_mmu_putpagetable(mmu->priv_bank_table);
mmu->priv_bank_table = NULL;
}
if (mmu->defaultpagetable) {
kgsl_mmu_putpagetable(mmu->defaultpagetable);
mmu->defaultpagetable = NULL;
}
return status;
}
