/*
* kgsl_iommu_default_setstate - Change the IOMMU pagetable or flush IOMMU tlb
* of the primary context bank
* @mmu - Pointer to mmu structure
* @flags - Flags indicating whether pagetable has to chnage or tlb is to be
* flushed or both
*
* Based on flags set the new pagetable fo the IOMMU unit or flush it's tlb or
* do both by doing direct register writes to the IOMMu registers through the
* cpu
* Return - void
*/
static void kgsl_iommu_default_setstate(struct kgsl_mmu *mmu,
uint32_t flags)
{
struct kgsl_iommu *iommu = mmu->priv;
int temp;
int i;
unsigned int pt_base = kgsl_iommu_get_pt_base_addr(mmu,
mmu->hwpagetable);
unsigned int pt_val;
if (kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER)) {
KGSL_DRV_ERR(mmu->device, "Failed to enable iommu clocks\n");
return;
}
/* Mask off the lsb of the pt base address since lsb will not change */
pt_base &= (iommu->iommu_reg_list[KGSL_IOMMU_CTX_TTBR0].reg_mask <<
iommu->iommu_reg_list[KGSL_IOMMU_CTX_TTBR0].reg_shift);
/* For v1 SMMU GPU needs to be idle for tlb invalidate as well */
if (msm_soc_version_supports_iommu_v1())
kgsl_idle(mmu->device);
/* Acquire GPU-CPU sync Lock here */
msm_iommu_lock();
if (flags & KGSL_MMUFLAGS_PTUPDATE) {
if (!msm_soc_version_supports_iommu_v1())
kgsl_idle(mmu->device);
for (i = 0; i < iommu->unit_count; i++) {
/* get the lsb value which should not change when
* changing ttbr0 */
pt_val = kgsl_iommu_get_pt_lsb(mmu, i,
KGSL_IOMMU_CONTEXT_USER);
pt_val += pt_base;
KGSL_IOMMU_SET_CTX_REG(iommu, (&iommu->iommu_units[i]),
KGSL_IOMMU_CONTEXT_USER, TTBR0, pt_val);
mb();
temp = KGSL_IOMMU_GET_CTX_REG(iommu,
(&iommu->iommu_units[i]),
KGSL_IOMMU_CONTEXT_USER, TTBR0);
}
}
/* Flush tlb */
if (flags & KGSL_MMUFLAGS_TLBFLUSH) {
for (i = 0; i < iommu->unit_count; i++) {
KGSL_IOMMU_SET_CTX_REG(iommu, (&iommu->iommu_units[i]),
KGSL_IOMMU_CONTEXT_USER, TLBIALL, 1);
mb();
}
}
/* Release GPU-CPU sync Lock here */
msm_iommu_unlock();
/* Disable smmu clock */
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
}
static void kgsl_iommu_default_setstate(struct kgsl_mmu *mmu,
uint32_t flags)
{
struct kgsl_iommu *iommu = mmu->priv;
int temp;
int i;
unsigned int pt_base = kgsl_iommu_pt_get_base_addr(
mmu->hwpagetable);
unsigned int pt_val;
if (kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER)) {
KGSL_DRV_ERR(mmu->device, "Failed to enable iommu clocks\n");
return;
}
pt_base &= (KGSL_IOMMU_TTBR0_PA_MASK << KGSL_IOMMU_TTBR0_PA_SHIFT);
if (msm_soc_version_supports_iommu_v1())
kgsl_idle(mmu->device, KGSL_TIMEOUT_DEFAULT);
msm_iommu_lock();
if (flags & KGSL_MMUFLAGS_PTUPDATE) {
if (!msm_soc_version_supports_iommu_v1())
kgsl_idle(mmu->device, KGSL_TIMEOUT_DEFAULT);
for (i = 0; i < iommu->unit_count; i++) {
pt_val = kgsl_iommu_get_pt_lsb(mmu, i,
KGSL_IOMMU_CONTEXT_USER);
pt_val += pt_base;
KGSL_IOMMU_SET_IOMMU_REG(
iommu->iommu_units[i].reg_map.hostptr,
KGSL_IOMMU_CONTEXT_USER, TTBR0, pt_val);
mb();
temp = KGSL_IOMMU_GET_IOMMU_REG(
iommu->iommu_units[i].reg_map.hostptr,
KGSL_IOMMU_CONTEXT_USER, TTBR0);
}
}
if (flags & KGSL_MMUFLAGS_TLBFLUSH) {
for (i = 0; i < iommu->unit_count; i++) {
KGSL_IOMMU_SET_IOMMU_REG(
iommu->iommu_units[i].reg_map.hostptr,
KGSL_IOMMU_CONTEXT_USER, CTX_TLBIALL,
1);
mb();
}
}
msm_iommu_unlock();
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
}
/*
* kgsl_iommu_setup_defaultpagetable - Setup the initial defualtpagetable
* for iommu. This function is only called once during first start, successive
* start do not call this funciton.
* @mmu - Pointer to mmu structure
*
* Create the initial defaultpagetable and setup the iommu mappings to it
* Return - 0 on success else error code
*/
static int kgsl_iommu_setup_defaultpagetable(struct kgsl_mmu *mmu)
{
int status = 0;
int i = 0;
struct kgsl_iommu *iommu = mmu->priv;
struct kgsl_pagetable *pagetable = NULL;
/* If chip is not 8960 then we use the 2nd context bank for pagetable
* switching on the 3D side for which a separate table is allocated */
if (!cpu_is_msm8960() && msm_soc_version_supports_iommu_v1()) {
mmu->priv_bank_table =
kgsl_mmu_getpagetable(KGSL_MMU_PRIV_BANK_TABLE_NAME);
if (mmu->priv_bank_table == NULL) {
status = -ENOMEM;
goto err;
}
}
mmu->defaultpagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
/* Return error if the default pagetable doesn't exist */
if (mmu->defaultpagetable == NULL) {
status = -ENOMEM;
goto err;
}
pagetable = mmu->priv_bank_table ? mmu->priv_bank_table :
mmu->defaultpagetable;
/* Map the IOMMU regsiters to only defaultpagetable */
if (msm_soc_version_supports_iommu_v1()) {
for (i = 0; i < iommu->unit_count; i++) {
iommu->iommu_units[i].reg_map.priv |=
KGSL_MEMFLAGS_GLOBAL;
status = kgsl_mmu_map(pagetable,
&(iommu->iommu_units[i].reg_map),
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (status) {
iommu->iommu_units[i].reg_map.priv &=
~KGSL_MEMFLAGS_GLOBAL;
goto err;
}
}
}
return status;
err:
for (i--; i >= 0; i--) {
kgsl_mmu_unmap(pagetable,
&(iommu->iommu_units[i].reg_map));
iommu->iommu_units[i].reg_map.priv &= ~KGSL_MEMFLAGS_GLOBAL;
}
if (mmu->priv_bank_table) {
kgsl_mmu_putpagetable(mmu->priv_bank_table);
mmu->priv_bank_table = NULL;
}
if (mmu->defaultpagetable) {
kgsl_mmu_putpagetable(mmu->defaultpagetable);
mmu->defaultpagetable = NULL;
}
return status;
}
/*
* kgsl_iommu_create_pagetable - Create a IOMMU pagetable
*
* Allocate memory to hold a pagetable and allocate the IOMMU
* domain which is the actual IOMMU pagetable
* Return - void
*/
void *kgsl_iommu_create_pagetable(void)
{
struct kgsl_iommu_pt *iommu_pt;
iommu_pt = kzalloc(sizeof(struct kgsl_iommu_pt), GFP_KERNEL);
if (!iommu_pt) {
KGSL_CORE_ERR("kzalloc(%d) failed\n",
sizeof(struct kgsl_iommu_pt));
return NULL;
}
/* L2 redirect is not stable on IOMMU v2 */
if (msm_soc_version_supports_iommu_v1())
iommu_pt->domain = iommu_domain_alloc(&platform_bus_type,
MSM_IOMMU_DOMAIN_PT_CACHEABLE);
else
iommu_pt->domain = iommu_domain_alloc(&platform_bus_type,
0);
if (!iommu_pt->domain) {
KGSL_CORE_ERR("Failed to create iommu domain\n");
kfree(iommu_pt);
return NULL;
} else {
iommu_set_fault_handler(iommu_pt->domain,
kgsl_iommu_fault_handler);
}
return iommu_pt;
}
static int
kgsl_iommu_unmap(void *mmu_specific_pt,
struct kgsl_memdesc *memdesc,
unsigned int *tlb_flags)
{
int ret;
unsigned int range = kgsl_sg_size(memdesc->sg, memdesc->sglen);
struct kgsl_iommu_pt *iommu_pt = mmu_specific_pt;
/* All GPU addresses as assigned are page aligned, but some
functions purturb the gpuaddr with an offset, so apply the
mask here to make sure we have the right address */
unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK;
if (range == 0 || gpuaddr == 0)
return 0;
ret = iommu_unmap_range(iommu_pt->domain, gpuaddr, range);
if (ret)
KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed "
"with err: %d\n", iommu_pt->domain, gpuaddr,
range, ret);
#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE
/*
* Flushing only required if per process pagetables are used. With
* global case, flushing will happen inside iommu_map function
*/
if (!ret && msm_soc_version_supports_iommu_v1())
*tlb_flags = UINT_MAX;
#endif
return 0;
}
static int __init iommu_init(void)
{
int ret;
if (!msm_soc_version_supports_iommu_v1()) {
pr_err("IOMMU v1 is not supported on this SoC version.\n");
return -ENODEV;
}
ret = platform_device_register(&msm_root_iommu_dev);
if (ret != 0) {
pr_err("Failed to register root IOMMU device!\n");
goto failure;
}
platform_add_devices(msm_iommu_common_devs,
ARRAY_SIZE(msm_iommu_common_devs));
if (cpu_is_msm8x60() || cpu_is_msm8960()) {
platform_add_devices(msm_iommu_jpegd_devs,
ARRAY_SIZE(msm_iommu_jpegd_devs));
platform_add_devices(msm_iommu_gfx2d_devs,
ARRAY_SIZE(msm_iommu_gfx2d_devs));
}
if (cpu_is_apq8064() || cpu_is_apq8064ab()) {
platform_add_devices(msm_iommu_jpegd_devs,
ARRAY_SIZE(msm_iommu_jpegd_devs));
platform_add_devices(msm_iommu_8064_devs,
ARRAY_SIZE(msm_iommu_8064_devs));
}
ret = platform_add_devices(msm_iommu_common_ctx_devs,
ARRAY_SIZE(msm_iommu_common_ctx_devs));
if (cpu_is_msm8x60() || cpu_is_msm8960()) {
platform_add_devices(msm_iommu_jpegd_ctx_devs,
ARRAY_SIZE(msm_iommu_jpegd_ctx_devs));
platform_add_devices(msm_iommu_gfx2d_ctx_devs,
ARRAY_SIZE(msm_iommu_gfx2d_ctx_devs));
}
if (cpu_is_apq8064() || cpu_is_apq8064ab()) {
platform_add_devices(msm_iommu_jpegd_ctx_devs,
ARRAY_SIZE(msm_iommu_jpegd_ctx_devs));
platform_add_devices(msm_iommu_8064_ctx_devs,
ARRAY_SIZE(msm_iommu_8064_ctx_devs));
}
return 0;
failure:
return ret;
}
static int __init iommu_init(void)
{
int ret;
if (!msm_soc_version_supports_iommu_v1()) {
pr_err("IOMMU v1 is not supported on this SoC version.\n");
return -ENODEV;
}
/* Initialize common devs */
platform_add_devices(msm_iommu_common_devs,
ARRAY_SIZE(msm_iommu_common_devs));
/* Initialize soc-specific devs */
if (cpu_is_msm8x60() || cpu_is_msm8960()) {
platform_add_devices(msm_iommu_jpegd_devs,
ARRAY_SIZE(msm_iommu_jpegd_devs));
platform_add_devices(msm_iommu_gfx2d_devs,
ARRAY_SIZE(msm_iommu_gfx2d_devs));
}
if (soc_class_is_apq8064() || cpu_is_msm8960ab()) {
platform_add_devices(msm_iommu_jpegd_devs,
ARRAY_SIZE(msm_iommu_jpegd_devs));
platform_add_devices(msm_iommu_adreno3xx_gfx_devs,
ARRAY_SIZE(msm_iommu_adreno3xx_gfx_devs));
}
if (soc_class_is_apq8064())
platform_add_devices(msm_iommu_vcap_devs,
ARRAY_SIZE(msm_iommu_vcap_devs));
/* Initialize common ctx_devs */
ret = platform_add_devices(msm_iommu_common_ctx_devs,
ARRAY_SIZE(msm_iommu_common_ctx_devs));
/* Initialize soc-specific ctx_devs */
if (cpu_is_msm8x60() || cpu_is_msm8960()) {
platform_add_devices(msm_iommu_jpegd_ctx_devs,
ARRAY_SIZE(msm_iommu_jpegd_ctx_devs));
platform_add_devices(msm_iommu_gfx2d_ctx_devs,
ARRAY_SIZE(msm_iommu_gfx2d_ctx_devs));
}
if (soc_class_is_apq8064() || cpu_is_msm8960ab()) {
platform_add_devices(msm_iommu_jpegd_ctx_devs,
ARRAY_SIZE(msm_iommu_jpegd_ctx_devs));
platform_add_devices(msm_iommu_adreno3xx_ctx_devs,
ARRAY_SIZE(msm_iommu_adreno3xx_ctx_devs));
}
if (soc_class_is_apq8064())
platform_add_devices(msm_iommu_vcap_ctx_devs,
ARRAY_SIZE(msm_iommu_vcap_ctx_devs));
return 0;
}
static int kgsl_iommu_init(struct kgsl_mmu *mmu)
{
/*
* intialize device mmu
*
* call this with the global lock held
*/
int status = 0;
struct kgsl_iommu *iommu;
iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL);
if (!iommu) {
KGSL_CORE_ERR("kzalloc(%d) failed\n",
sizeof(struct kgsl_iommu));
return -ENOMEM;
}
mmu->priv = iommu;
status = kgsl_get_iommu_ctxt(mmu);
if (status)
goto done;
status = kgsl_set_register_map(mmu);
if (status)
goto done;
iommu->iommu_reg_list = kgsl_iommuv1_reg;
iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1;
if (msm_soc_version_supports_iommu_v1()) {
iommu->iommu_reg_list = kgsl_iommuv1_reg;
iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1;
} else {
iommu->iommu_reg_list = kgsl_iommuv2_reg;
iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V2;
}
/* A nop is required in an indirect buffer when switching
* pagetables in-stream */
kgsl_sharedmem_writel(&mmu->setstate_memory,
KGSL_IOMMU_SETSTATE_NOP_OFFSET,
cp_nop_packet(1));
dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n",
__func__);
done:
if (status) {
kfree(iommu);
mmu->priv = NULL;
}
return status;
}
static int kgsl_iommu_start(struct kgsl_mmu *mmu)
{
struct kgsl_device *device = mmu->device;
int status;
struct kgsl_iommu *iommu = mmu->priv;
int i, j;
if (mmu->flags & KGSL_FLAGS_STARTED)
return 0;
if (mmu->defaultpagetable == NULL) {
status = kgsl_iommu_setup_defaultpagetable(mmu);
if (status)
return -ENOMEM;
/* Initialize the sync lock between GPU and CPU */
if (msm_soc_version_supports_iommu_v1() &&
(device->id == KGSL_DEVICE_3D0))
kgsl_iommu_init_sync_lock(mmu);
}
/* We use the GPU MMU to control access to IOMMU registers on 8960 with
* a225, hence we still keep the MMU active on 8960 */
if (cpu_is_msm8960()) {
struct kgsl_mh *mh = &(mmu->device->mh);
kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000001);
kgsl_regwrite(mmu->device, MH_MMU_MPU_END,
mh->mpu_base +
iommu->iommu_units[0].reg_map.gpuaddr);
} else {
kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000000);
}
mmu->hwpagetable = mmu->defaultpagetable;
status = kgsl_attach_pagetable_iommu_domain(mmu);
if (status) {
mmu->hwpagetable = NULL;
goto done;
}
status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER);
if (status) {
KGSL_CORE_ERR("clk enable failed\n");
goto done;
}
status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_PRIV);
if (status) {
KGSL_CORE_ERR("clk enable failed\n");
goto done;
}
/* Get the lsb value of pagetables set in the IOMMU ttbr0 register as
* that value should not change when we change pagetables, so while
* changing pagetables we can use this lsb value of the pagetable w/o
* having to read it again
*/
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
for (j = 0; j < iommu_unit->dev_count; j++)
iommu_unit->dev[j].pt_lsb = KGSL_IOMMMU_PT_LSB(iommu,
KGSL_IOMMU_GET_CTX_REG(iommu,
iommu_unit,
iommu_unit->dev[j].ctx_id,
TTBR0));
}
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
mmu->flags |= KGSL_FLAGS_STARTED;
done:
if (status) {
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
kgsl_detach_pagetable_iommu_domain(mmu);
}
return status;
}
static int kgsl_iommu_start(struct kgsl_mmu *mmu)
{
struct kgsl_device *device = mmu->device;
int status;
struct kgsl_iommu *iommu = mmu->priv;
int i, j;
if (mmu->flags & KGSL_FLAGS_STARTED)
return 0;
if (mmu->defaultpagetable == NULL) {
status = kgsl_iommu_setup_defaultpagetable(mmu);
if (status)
return -ENOMEM;
if (msm_soc_version_supports_iommu_v1() &&
(device->id == KGSL_DEVICE_3D0))
kgsl_iommu_init_sync_lock(mmu);
}
if (cpu_is_msm8960()) {
struct kgsl_mh *mh = &(mmu->device->mh);
kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000001);
kgsl_regwrite(mmu->device, MH_MMU_MPU_END,
mh->mpu_base +
iommu->iommu_units
[iommu->unit_count - 1].reg_map.gpuaddr -
PAGE_SIZE);
} else {
kgsl_regwrite(mmu->device, MH_MMU_CONFIG, 0x00000000);
}
mmu->hwpagetable = mmu->defaultpagetable;
status = kgsl_attach_pagetable_iommu_domain(mmu);
if (status) {
mmu->hwpagetable = NULL;
goto done;
}
status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_USER);
if (status) {
KGSL_CORE_ERR("clk enable failed\n");
goto done;
}
status = kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_CONTEXT_PRIV);
if (status) {
KGSL_CORE_ERR("clk enable failed\n");
goto done;
}
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
for (j = 0; j < iommu_unit->dev_count; j++)
iommu_unit->dev[j].pt_lsb = KGSL_IOMMMU_PT_LSB(
KGSL_IOMMU_GET_IOMMU_REG(
iommu_unit->reg_map.hostptr,
iommu_unit->dev[j].ctx_id,
TTBR0));
}
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
mmu->flags |= KGSL_FLAGS_STARTED;
done:
if (status) {
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
kgsl_detach_pagetable_iommu_domain(mmu);
}
return status;
}
