/* * kgsl_iommu_default_setstate - Change the IOMMU pagetable or flush IOMMU tlb * of the primary context bank * @mmu - Pointer to mmu structure * @flags - Flags indicating whether pagetable has to chnage or tlb is to be * flushed or both * * Based on flags set the new pagetable fo the IOMMU unit or flush it's tlb or * do both by doing direct register writes to the IOMMu registers through the * cpu * Return - void */ static void kgsl_iommu_default_setstate(struct kgsl_mmu *mmu, uint32_t flags) { struct kgsl_iommu *iommu = mmu->priv; int temp; int i; unsigned int pt_base = kgsl_iommu_get_pt_base_addr(mmu, mmu->hwpagetable); unsigned int pt_val; if (kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER)) { KGSL_DRV_ERR(mmu->device, "Failed to enable iommu clocks\n"); return; } /* Mask off the lsb of the pt base address since lsb will not change */ pt_base &= (iommu->iommu_reg_list[KGSL_IOMMU_CTX_TTBR0].reg_mask << iommu->iommu_reg_list[KGSL_IOMMU_CTX_TTBR0].reg_shift); /* For v1 SMMU GPU needs to be idle for tlb invalidate as well */ if (msm_soc_version_supports_iommu_v1()) kgsl_idle(mmu->device); /* Acquire GPU-CPU sync Lock here */ msm_iommu_lock(); if (flags & KGSL_MMUFLAGS_PTUPDATE) { if (!msm_soc_version_supports_iommu_v1()) kgsl_idle(mmu->device); for (i = 0; i < iommu->unit_count; i++) { /* get the lsb value which should not change when * changing ttbr0 */ pt_val = kgsl_iommu_get_pt_lsb(mmu, i, KGSL_IOMMU_CONTEXT_USER); pt_val += pt_base; KGSL_IOMMU_SET_CTX_REG(iommu, (&iommu->iommu_units[i]), KGSL_IOMMU_CONTEXT_USER, TTBR0, pt_val); mb(); temp = KGSL_IOMMU_GET_CTX_REG(iommu, (&iommu->iommu_units[i]), KGSL_IOMMU_CONTEXT_USER, TTBR0); } } /* Flush tlb */ if (flags & KGSL_MMUFLAGS_TLBFLUSH) { for (i = 0; i < iommu->unit_count; i++) { KGSL_IOMMU_SET_CTX_REG(iommu, (&iommu->iommu_units[i]), KGSL_IOMMU_CONTEXT_USER, TLBIALL, 1); mb(); } } /* Release GPU-CPU sync Lock here */ msm_iommu_unlock(); /* Disable smmu clock */ kgsl_iommu_disable_clk_on_ts(mmu, 0, false); }
static void kgsl_iommu_default_setstate(struct kgsl_mmu *mmu, uint32_t flags) { struct kgsl_iommu *iommu = mmu->priv; int temp; int i; unsigned int pt_base = kgsl_iommu_pt_get_base_addr( mmu->hwpagetable); unsigned int pt_val; if (kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER)) { KGSL_DRV_ERR(mmu->device, "Failed to enable iommu clocks\n"); return; } pt_base &= (KGSL_IOMMU_TTBR0_PA_MASK << KGSL_IOMMU_TTBR0_PA_SHIFT); if (msm_soc_version_supports_iommu_v1()) kgsl_idle(mmu->device, KGSL_TIMEOUT_DEFAULT); msm_iommu_lock(); if (flags & KGSL_MMUFLAGS_PTUPDATE) { if (!msm_soc_version_supports_iommu_v1()) kgsl_idle(mmu->device, KGSL_TIMEOUT_DEFAULT); for (i = 0; i < iommu->unit_count; i++) { pt_val = kgsl_iommu_get_pt_lsb(mmu, i, KGSL_IOMMU_CONTEXT_USER); pt_val += pt_base; KGSL_IOMMU_SET_IOMMU_REG( iommu->iommu_units[i].reg_map.hostptr, KGSL_IOMMU_CONTEXT_USER, TTBR0, pt_val); mb(); temp = KGSL_IOMMU_GET_IOMMU_REG( iommu->iommu_units[i].reg_map.hostptr, KGSL_IOMMU_CONTEXT_USER, TTBR0); } } if (flags & KGSL_MMUFLAGS_TLBFLUSH) { for (i = 0; i < iommu->unit_count; i++) { KGSL_IOMMU_SET_IOMMU_REG( iommu->iommu_units[i].reg_map.hostptr, KGSL_IOMMU_CONTEXT_USER, CTX_TLBIALL, 1); mb(); } } msm_iommu_unlock(); kgsl_iommu_disable_clk_on_ts(mmu, 0, false); }
/* * kgsl_iommu_setup_defaultpagetable - Setup the initial defualtpagetable * for iommu. This function is only called once during first start, successive * start do not call this funciton. * @mmu - Pointer to mmu structure * * Create the initial defaultpagetable and setup the iommu mappings to it * Return - 0 on success else error code */ static int kgsl_iommu_setup_defaultpagetable(struct kgsl_mmu *mmu) { int status = 0; int i = 0; struct kgsl_iommu *iommu = mmu->priv; struct kgsl_pagetable *pagetable = NULL; /* If chip is not 8960 then we use the 2nd context bank for pagetable * switching on the 3D side for which a separate table is allocated */ if (!cpu_is_msm8960() && msm_soc_version_supports_iommu_v1()) { mmu->priv_bank_table = kgsl_mmu_getpagetable(KGSL_MMU_PRIV_BANK_TABLE_NAME); if (mmu->priv_bank_table == NULL) { status = -ENOMEM; goto err; } } mmu->defaultpagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT); /* Return error if the default pagetable doesn't exist */ if (mmu->defaultpagetable == NULL) { status = -ENOMEM; goto err; } pagetable = mmu->priv_bank_table ? mmu->priv_bank_table : mmu->defaultpagetable; /* Map the IOMMU regsiters to only defaultpagetable */ if (msm_soc_version_supports_iommu_v1()) { for (i = 0; i < iommu->unit_count; i++) { iommu->iommu_units[i].reg_map.priv |= KGSL_MEMFLAGS_GLOBAL; status = kgsl_mmu_map(pagetable, &(iommu->iommu_units[i].reg_map), GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); if (status) { iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL; goto err; } } } return status; err: for (i--; i >= 0; i--) { kgsl_mmu_unmap(pagetable, &(iommu->iommu_units[i].reg_map)); iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL; } if (mmu->priv_bank_table) { kgsl_mmu_putpagetable(mmu->priv_bank_table); mmu->priv_bank_table = NULL; } if (mmu->defaultpagetable) { kgsl_mmu_putpagetable(mmu->defaultpagetable); mmu->defaultpagetable = NULL; } return status; }
/* * kgsl_iommu_create_pagetable - Create a IOMMU pagetable * * Allocate memory to hold a pagetable and allocate the IOMMU * domain which is the actual IOMMU pagetable * Return - void */ void *kgsl_iommu_create_pagetable(void) { struct kgsl_iommu_pt *iommu_pt; iommu_pt = kzalloc(sizeof(struct kgsl_iommu_pt), GFP_KERNEL); if (!iommu_pt) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu_pt)); return NULL; } /* L2 redirect is not stable on IOMMU v2 */ if (msm_soc_version_supports_iommu_v1()) iommu_pt->domain = iommu_domain_alloc(&platform_bus_type, MSM_IOMMU_DOMAIN_PT_CACHEABLE); else iommu_pt->domain = iommu_domain_alloc(&platform_bus_type, 0); if (!iommu_pt->domain) { KGSL_CORE_ERR("Failed to create iommu domain\n"); kfree(iommu_pt); return NULL; } else { iommu_set_fault_handler(iommu_pt->domain, kgsl_iommu_fault_handler); } return iommu_pt; }
static int kgsl_iommu_unmap(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int *tlb_flags) { int ret; unsigned int range = kgsl_sg_size(memdesc->sg, memdesc->sglen); struct kgsl_iommu_pt *iommu_pt = mmu_specific_pt; /* All GPU addresses as assigned are page aligned, but some functions purturb the gpuaddr with an offset, so apply the mask here to make sure we have the right address */ unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK; if (range == 0 || gpuaddr == 0) return 0; ret = iommu_unmap_range(iommu_pt->domain, gpuaddr, range); if (ret) KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed " "with err: %d\n", iommu_pt->domain, gpuaddr, range, ret); #ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE /* * Flushing only required if per process pagetables are used. With * global case, flushing will happen inside iommu_map function */ if (!ret && msm_soc_version_supports_iommu_v1()) *tlb_flags = UINT_MAX; #endif return 0; }
static int __init iommu_init(void) { int ret; if (!msm_soc_version_supports_iommu_v1()) { pr_err("IOMMU v1 is not supported on this SoC version.\n"); return -ENODEV; } ret = platform_device_register(&msm_root_iommu_dev); if (ret != 0) { pr_err("Failed to register root IOMMU device!\n"); goto failure; } platform_add_devices(msm_iommu_common_devs, ARRAY_SIZE(msm_iommu_common_devs)); if (cpu_is_msm8x60() || cpu_is_msm8960()) { platform_add_devices(msm_iommu_jpegd_devs, ARRAY_SIZE(msm_iommu_jpegd_devs)); platform_add_devices(msm_iommu_gfx2d_devs, ARRAY_SIZE(msm_iommu_gfx2d_devs)); } if (cpu_is_apq8064() || cpu_is_apq8064ab()) { platform_add_devices(msm_iommu_jpegd_devs, ARRAY_SIZE(msm_iommu_jpegd_devs)); platform_add_devices(msm_iommu_8064_devs, ARRAY_SIZE(msm_iommu_8064_devs)); } ret = platform_add_devices(msm_iommu_common_ctx_devs, ARRAY_SIZE(msm_iommu_common_ctx_devs)); if (cpu_is_msm8x60() || cpu_is_msm8960()) { platform_add_devices(msm_iommu_jpegd_ctx_devs, ARRAY_SIZE(msm_iommu_jpegd_ctx_devs)); platform_add_devices(msm_iommu_gfx2d_ctx_devs, ARRAY_SIZE(msm_iommu_gfx2d_ctx_devs)); } if (cpu_is_apq8064() || cpu_is_apq8064ab()) { platform_add_devices(msm_iommu_jpegd_ctx_devs, ARRAY_SIZE(msm_iommu_jpegd_ctx_devs)); platform_add_devices(msm_iommu_8064_ctx_devs, ARRAY_SIZE(msm_iommu_8064_ctx_devs)); } return 0; failure: return ret; }
static int __init iommu_init(void) { int ret; if (!msm_soc_version_supports_iommu_v1()) { pr_err("IOMMU v1 is not supported on this SoC version.\n"); return -ENODEV; } /* Initialize common devs */ platform_add_devices(msm_iommu_common_devs, ARRAY_SIZE(msm_iommu_common_devs)); /* Initialize soc-specific devs */ if (cpu_is_msm8x60() || cpu_is_msm8960()) { platform_add_devices(msm_iommu_jpegd_devs, ARRAY_SIZE(msm_iommu_jpegd_devs)); platform_add_devices(msm_iommu_gfx2d_devs, ARRAY_SIZE(msm_iommu_gfx2d_devs)); } if (soc_class_is_apq8064() || cpu_is_msm8960ab()) { platform_add_devices(msm_iommu_jpegd_devs, ARRAY_SIZE(msm_iommu_jpegd_devs)); platform_add_devices(msm_iommu_adreno3xx_gfx_devs, ARRAY_SIZE(msm_iommu_adreno3xx_gfx_devs)); } if (soc_class_is_apq8064()) platform_add_devices(msm_iommu_vcap_devs, ARRAY_SIZE(msm_iommu_vcap_devs)); /* Initialize common ctx_devs */ ret = platform_add_devices(msm_iommu_common_ctx_devs, ARRAY_SIZE(msm_iommu_common_ctx_devs)); /* Initialize soc-specific ctx_devs */ if (cpu_is_msm8x60() || cpu_is_msm8960()) { platform_add_devices(msm_iommu_jpegd_ctx_devs, ARRAY_SIZE(msm_iommu_jpegd_ctx_devs)); platform_add_devices(msm_iommu_gfx2d_ctx_devs, ARRAY_SIZE(msm_iommu_gfx2d_ctx_devs)); } if (soc_class_is_apq8064() || cpu_is_msm8960ab()) { platform_add_devices(msm_iommu_jpegd_ctx_devs, ARRAY_SIZE(msm_iommu_jpegd_ctx_devs)); platform_add_devices(msm_iommu_adreno3xx_ctx_devs, ARRAY_SIZE(msm_iommu_adreno3xx_ctx_devs)); } if (soc_class_is_apq8064()) platform_add_devices(msm_iommu_vcap_ctx_devs, ARRAY_SIZE(msm_iommu_vcap_ctx_devs)); return 0; }
static int kgsl_iommu_init(struct kgsl_mmu *mmu) { /* * intialize device mmu * * call this with the global lock held */ int status = 0; struct kgsl_iommu *iommu; iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL); if (!iommu) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); return -ENOMEM; } mmu->priv = iommu; status = kgsl_get_iommu_ctxt(mmu); if (status) goto done; status = kgsl_set_register_map(mmu); if (status) goto done; iommu->iommu_reg_list = kgsl_iommuv1_reg; iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1; if (msm_soc_version_supports_iommu_v1()) { iommu->iommu_reg_list = kgsl_iommuv1_reg; iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1; } else { iommu->iommu_reg_list = kgsl_iommuv2_reg; iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V2; } /* A nop is required in an indirect buffer when switching * pagetables in-stream */ kgsl_sharedmem_writel(&mmu->setstate_memory, KGSL_IOMMU_SETSTATE_NOP_OFFSET, cp_nop_packet(1)); dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n", __func__); done: if (status) { kfree(iommu); mmu->priv = NULL; } return status; }
static int kgsl_iommu_start(struct kgsl_mmu *mmu) { struct kgsl_device *device = mmu->device; int status; struct kgsl_iommu *iommu = mmu->priv; int i, j; if (mmu->flags & KGSL_FLAGS_STARTED) return 0; if (mmu->defaultpagetable == NULL) { status = kgsl_iommu_setup_defaultpagetable(mmu); if (status) return -ENOMEM; /* Initialize the sync lock between GPU and CPU */ if (msm_soc_version_supports_iommu_v1() && (device->id == KGSL_DEVICE_3D0)) kgsl_iommu_init_sync_lock(mmu); } /* We use the GPU MMU to control access to IOMMU registers on 8960 with * a225, hence we still keep the MMU active on 8960 */ if (cpu_is_msm8960()) { struct kgsl_mh *mh = &(mmu->device->mh); kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000001); kgsl_regwrite(mmu->device, MH_MMU_MPU_END, mh->mpu_base + iommu->iommu_units[0].reg_map.gpuaddr); } else { kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000000); } mmu->hwpagetable = mmu->defaultpagetable; status = kgsl_attach_pagetable_iommu_domain(mmu); if (status) { mmu->hwpagetable = NULL; goto done; } status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER); if (status) { KGSL_CORE_ERR("clk enable failed\n"); goto done; } status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_PRIV); if (status) { KGSL_CORE_ERR("clk enable failed\n"); goto done; } /* Get the lsb value of pagetables set in the IOMMU ttbr0 register as * that value should not change when we change pagetables, so while * changing pagetables we can use this lsb value of the pagetable w/o * having to read it again */ for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) iommu_unit->dev[j].pt_lsb = KGSL_IOMMMU_PT_LSB(iommu, KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, iommu_unit->dev[j].ctx_id, TTBR0)); } kgsl_iommu_disable_clk_on_ts(mmu, 0, false); mmu->flags |= KGSL_FLAGS_STARTED; done: if (status) { kgsl_iommu_disable_clk_on_ts(mmu, 0, false); kgsl_detach_pagetable_iommu_domain(mmu); } return status; }
static int kgsl_iommu_start(struct kgsl_mmu *mmu) { struct kgsl_device *device = mmu->device; int status; struct kgsl_iommu *iommu = mmu->priv; int i, j; if (mmu->flags & KGSL_FLAGS_STARTED) return 0; if (mmu->defaultpagetable == NULL) { status = kgsl_iommu_setup_defaultpagetable(mmu); if (status) return -ENOMEM; if (msm_soc_version_supports_iommu_v1() && (device->id == KGSL_DEVICE_3D0)) kgsl_iommu_init_sync_lock(mmu); } if (cpu_is_msm8960()) { struct kgsl_mh *mh = &(mmu->device->mh); kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000001); kgsl_regwrite(mmu->device, MH_MMU_MPU_END, mh->mpu_base + iommu->iommu_units [iommu->unit_count - 1].reg_map.gpuaddr - PAGE_SIZE); } else { kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000000); } mmu->hwpagetable = mmu->defaultpagetable; status = kgsl_attach_pagetable_iommu_domain(mmu); if (status) { mmu->hwpagetable = NULL; goto done; } status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER); if (status) { KGSL_CORE_ERR("clk enable failed\n"); goto done; } status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_PRIV); if (status) { KGSL_CORE_ERR("clk enable failed\n"); goto done; } for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) iommu_unit->dev[j].pt_lsb = KGSL_IOMMMU_PT_LSB( KGSL_IOMMU_GET_IOMMU_REG( iommu_unit->reg_map.hostptr, iommu_unit->dev[j].ctx_id, TTBR0)); } kgsl_iommu_disable_clk_on_ts(mmu, 0, false); mmu->flags |= KGSL_FLAGS_STARTED; done: if (status) { kgsl_iommu_disable_clk_on_ts(mmu, 0, false); kgsl_detach_pagetable_iommu_domain(mmu); } return status; }