static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct iommu_domain *domain; struct kgsl_iommu *iommu = mmu->priv; BUG_ON(mmu->hwpagetable == NULL); BUG_ON(mmu->hwpagetable->priv == NULL); domain = mmu->hwpagetable->priv; if (iommu->iommu_user_dev_attached) { iommu_detach_device(domain, iommu->iommu_user_dev); iommu->iommu_user_dev_attached = 0; KGSL_MEM_INFO(mmu->device, "iommu %p detached from user dev of MMU: %p\n", domain, mmu); } if (iommu->iommu_priv_dev_attached) { iommu_detach_device(domain, iommu->iommu_priv_dev); iommu->iommu_priv_dev_attached = 0; KGSL_MEM_INFO(mmu->device, "iommu %p detached from priv dev of MMU: %p\n", domain, mmu); } }
/* * kgsl_detach_pagetable_iommu_domain - Detach the IOMMU unit from a * pagetable * @mmu - Pointer to the device mmu structure * @priv - Flag indicating whether the private or user context is to be * detached * * Detach the IOMMU unit with the domain that is contained in the * hwpagetable of the given mmu. After detaching the IOMMU unit is not * in use because the PTBR will not be set after a detach * Return - void */ static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct kgsl_iommu_pt *iommu_pt; struct kgsl_iommu *iommu = mmu->priv; int i, j; for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; iommu_pt = mmu->defaultpagetable->priv; for (j = 0; j < iommu_unit->dev_count; j++) { /* * If there is a 2nd default pagetable then priv domain * is attached with this pagetable */ if (mmu->priv_bank_table && (KGSL_IOMMU_CONTEXT_PRIV == j)) iommu_pt = mmu->priv_bank_table->priv; if (iommu_unit->dev[j].attached) { iommu_detach_device(iommu_pt->domain, iommu_unit->dev[j].dev); iommu_unit->dev[j].attached = false; KGSL_MEM_INFO(mmu->device, "iommu %p detached " "from user dev of MMU: %p\n", iommu_pt->domain, mmu); } } } }
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct kgsl_iommu_pt *iommu_pt; struct kgsl_iommu *iommu = mmu->priv; int i, j, ret = 0; for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; iommu_pt = mmu->defaultpagetable->priv; for (j = 0; j < iommu_unit->dev_count; j++) { if (mmu->priv_bank_table && (KGSL_IOMMU_CONTEXT_PRIV == j)) iommu_pt = mmu->priv_bank_table->priv; if (!iommu_unit->dev[j].attached) { ret = iommu_attach_device(iommu_pt->domain, iommu_unit->dev[j].dev); if (ret) { KGSL_MEM_ERR(mmu->device, "Failed to attach device, err %d\n", ret); goto done; } iommu_unit->dev[j].attached = true; KGSL_MEM_INFO(mmu->device, "iommu pt %p attached to dev %p, ctx_id %d\n", iommu_pt->domain, iommu_unit->dev[j].dev, iommu_unit->dev[j].ctx_id); } } } done: return ret; }
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct iommu_domain *domain; struct kgsl_iommu *iommu = mmu->priv; int i, ret = 0; BUG_ON(mmu->hwpagetable == NULL); BUG_ON(mmu->hwpagetable->priv == NULL); domain = mmu->hwpagetable->priv; for (i = 0; i < iommu->dev_count; i++) { if (iommu->dev[i].attached == 0) { ret = iommu_attach_device(domain, iommu->dev[i].dev); if (ret) { KGSL_MEM_ERR(mmu->device, "Failed to attach device, err %d\n", ret); goto done; } iommu->dev[i].attached = 1; KGSL_MEM_INFO(mmu->device, "iommu %p detached from user dev of MMU: %p\n", domain, mmu); } } done: return ret; }
/* * kgsl_detach_pagetable_iommu_domain - Detach the IOMMU unit from a * pagetable * @mmu - Pointer to the device mmu structure * @priv - Flag indicating whether the private or user context is to be * detached * * Detach the IOMMU unit with the domain that is contained in the * hwpagetable of the given mmu. After detaching the IOMMU unit is not * in use because the PTBR will not be set after a detach * Return - void */ static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct kgsl_iommu_pt *iommu_pt; struct kgsl_iommu *iommu = mmu->priv; int i, j; BUG_ON(mmu->hwpagetable == NULL); BUG_ON(mmu->hwpagetable->priv == NULL); iommu_pt = mmu->hwpagetable->priv; for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) { if (iommu_unit->dev[j].attached) { iommu_detach_device(iommu_pt->domain, iommu_unit->dev[j].dev); iommu_unit->dev[j].attached = false; KGSL_MEM_INFO(mmu->device, "iommu %p detached " "from user dev of MMU: %p\n", iommu_pt->domain, mmu); } } } }
int kgsl_mmu_setstate(struct kgsl_device *device, struct kgsl_pagetable *pagetable) { int status = 0; struct kgsl_mmu *mmu = &device->mmu; KGSL_MEM_VDBG("enter (device=%p, pagetable=%p)\n", device, pagetable); if (mmu->flags & KGSL_FLAGS_STARTED) { /* page table not current, then setup mmu to use new * specified page table */ KGSL_MEM_INFO("from %p to %p\n", mmu->hwpagetable, pagetable); if (mmu->hwpagetable != pagetable) { mmu->hwpagetable = pagetable; /* call device specific set page table */ status = kgsl_yamato_setstate(mmu->device, KGSL_MMUFLAGS_TLBFLUSH | KGSL_MMUFLAGS_PTUPDATE); } } KGSL_MEM_VDBG("return %d\n", status); return status; }
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct iommu_domain *domain; int ret = 0; struct kgsl_iommu *iommu = mmu->priv; BUG_ON(mmu->hwpagetable == NULL); BUG_ON(mmu->hwpagetable->priv == NULL); domain = mmu->hwpagetable->priv; if (iommu->iommu_user_dev && !iommu->iommu_user_dev_attached) { ret = iommu_attach_device(domain, iommu->iommu_user_dev); if (ret) { KGSL_MEM_ERR(mmu->device, "Failed to attach device, err %d\n", ret); goto done; } iommu->iommu_user_dev_attached = 1; KGSL_MEM_INFO(mmu->device, "iommu %p attached to user dev of MMU: %p\n", domain, mmu); } if (iommu->iommu_priv_dev && !iommu->iommu_priv_dev_attached) { ret = iommu_attach_device(domain, iommu->iommu_priv_dev); if (ret) { KGSL_MEM_ERR(mmu->device, "Failed to attach device, err %d\n", ret); iommu_detach_device(domain, iommu->iommu_user_dev); iommu->iommu_user_dev_attached = 0; goto done; } iommu->iommu_priv_dev_attached = 1; KGSL_MEM_INFO(mmu->device, "iommu %p attached to priv dev of MMU: %p\n", domain, mmu); } done: return ret; }
static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct iommu_domain *domain; struct kgsl_iommu *iommu = mmu->priv; int i; BUG_ON(mmu->hwpagetable == NULL); BUG_ON(mmu->hwpagetable->priv == NULL); domain = mmu->hwpagetable->priv; for (i = 0; i < iommu->dev_count; i++) { iommu_detach_device(domain, iommu->dev[i].dev); iommu->dev[i].attached = 0; KGSL_MEM_INFO(mmu->device, "iommu %p detached from user dev of MMU: %p\n", domain, mmu); } }
/* * kgsl_attach_pagetable_iommu_domain - Attach the IOMMU unit to a * pagetable, i.e set the IOMMU's PTBR to the pagetable address and * setup other IOMMU registers for the device so that it becomes * active * @mmu - Pointer to the device mmu structure * @priv - Flag indicating whether the private or user context is to be * attached * * Attach the IOMMU unit with the domain that is contained in the * hwpagetable of the given mmu. * Return - 0 on success else error code */ static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct kgsl_iommu_pt *iommu_pt; struct kgsl_iommu *iommu = mmu->priv; int i, j, ret = 0; /* * Loop through all the iommu devcies under all iommu units and * attach the domain */ for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; iommu_pt = mmu->defaultpagetable->priv; for (j = 0; j < iommu_unit->dev_count; j++) { /* * If there is a 2nd default pagetable then priv domain * is attached to this pagetable */ if (mmu->priv_bank_table && (KGSL_IOMMU_CONTEXT_PRIV == j)) iommu_pt = mmu->priv_bank_table->priv; if (!iommu_unit->dev[j].attached) { ret = iommu_attach_device(iommu_pt->domain, iommu_unit->dev[j].dev); if (ret) { KGSL_MEM_ERR(mmu->device, "Failed to attach device, err %d\n", ret); goto done; } iommu_unit->dev[j].attached = true; KGSL_MEM_INFO(mmu->device, "iommu pt %p attached to dev %p, ctx_id %d\n", iommu_pt->domain, iommu_unit->dev[j].dev, iommu_unit->dev[j].ctx_id); } } } done: return ret; }
/* * kgsl_attach_pagetable_iommu_domain - Attach the IOMMU unit to a * pagetable, i.e set the IOMMU's PTBR to the pagetable address and * setup other IOMMU registers for the device so that it becomes * active * @mmu - Pointer to the device mmu structure * @priv - Flag indicating whether the private or user context is to be * attached * * Attach the IOMMU unit with the domain that is contained in the * hwpagetable of the given mmu. * Return - 0 on success else error code */ static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) { struct kgsl_iommu_pt *iommu_pt; struct kgsl_iommu *iommu = mmu->priv; int i, j, ret = 0; BUG_ON(mmu->hwpagetable == NULL); BUG_ON(mmu->hwpagetable->priv == NULL); iommu_pt = mmu->hwpagetable->priv; /* * Loop through all the iommu devcies under all iommu units and * attach the domain */ for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) { if (!iommu_unit->dev[j].attached) { ret = iommu_attach_device(iommu_pt->domain, iommu_unit->dev[j].dev); if (ret) { KGSL_MEM_ERR(mmu->device, "Failed to attach device, err %d\n", ret); goto done; } iommu_unit->dev[j].attached = true; KGSL_MEM_INFO(mmu->device, "iommu pt %p attached to dev %p, ctx_id %d\n", iommu_pt->domain, iommu_unit->dev[j].dev, iommu_unit->dev[j].ctx_id); } } } done: return ret; }
int kgsl_mmu_init(struct kgsl_device *device) { /* * intialize device mmu * * call this with the global lock held */ int status; uint32_t flags; struct kgsl_mmu *mmu = &device->mmu; #ifdef _DEBUG struct kgsl_mmu_debug regs; #endif /* _DEBUG */ KGSL_MEM_VDBG("enter (device=%p)\n", device); if (mmu->flags & KGSL_FLAGS_INITIALIZED0) { KGSL_MEM_INFO("MMU already initialized.\n"); return 0; } mmu->device = device; #ifndef CONFIG_MSM_KGSL_MMU mmu->config = 0x00000000; #endif /* setup MMU and sub-client behavior */ kgsl_yamato_regwrite(device, REG_MH_MMU_CONFIG, mmu->config); /* enable axi interrupts */ KGSL_MEM_DBG("enabling mmu interrupts mask=0x%08lx\n", GSL_MMU_INT_MASK); kgsl_yamato_regwrite(device, REG_MH_INTERRUPT_MASK, GSL_MMU_INT_MASK); mmu->flags |= KGSL_FLAGS_INITIALIZED0; /* MMU not enabled */ if ((mmu->config & 0x1) == 0) { KGSL_MEM_VDBG("return %d\n", 0); return 0; } /* idle device */ kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT); /* make sure aligned to pagesize */ BUG_ON(mmu->mpu_base & (KGSL_PAGESIZE - 1)); BUG_ON((mmu->mpu_base + mmu->mpu_range) & (KGSL_PAGESIZE - 1)); /* define physical memory range accessible by the core */ kgsl_yamato_regwrite(device, REG_MH_MMU_MPU_BASE, mmu->mpu_base); kgsl_yamato_regwrite(device, REG_MH_MMU_MPU_END, mmu->mpu_base + mmu->mpu_range); /* enable axi interrupts */ KGSL_MEM_DBG("enabling mmu interrupts mask=0x%08lx\n", GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT); kgsl_yamato_regwrite(device, REG_MH_INTERRUPT_MASK, GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT); mmu->flags |= KGSL_FLAGS_INITIALIZED; /* sub-client MMU lookups require address translation */ if ((mmu->config & ~0x1) > 0) { /*make sure virtual address range is a multiple of 64Kb */ BUG_ON(mmu->va_range & ((1 << 16) - 1)); /* allocate memory used for completing r/w operations that * cannot be mapped by the MMU */ flags = (KGSL_MEMFLAGS_ALIGN4K | KGSL_MEMFLAGS_CONPHYS | KGSL_MEMFLAGS_STRICTREQUEST); status = kgsl_sharedmem_alloc(flags, 64, &mmu->dummyspace); if (status != 0) { KGSL_MEM_ERR ("Unable to allocate dummy space memory.\n"); kgsl_mmu_close(device); return status; } kgsl_sharedmem_set(&mmu->dummyspace, 0, 0, mmu->dummyspace.size); /* TRAN_ERROR needs a 32 byte (32 byte aligned) chunk of memory * to complete transactions in case of an MMU fault. Note that * we'll leave the bottom 32 bytes of the dummyspace for other * purposes (e.g. use it when dummy read cycles are needed * for other blocks */ kgsl_yamato_regwrite(device, REG_MH_MMU_TRAN_ERROR, mmu->dummyspace.physaddr + 32); mmu->defaultpagetable = kgsl_mmu_createpagetableobject(mmu); if (!mmu->defaultpagetable) { KGSL_MEM_ERR("Failed to create global page table\n"); kgsl_mmu_close(device); return -ENOMEM; } mmu->hwpagetable = mmu->defaultpagetable; kgsl_yamato_regwrite(device, REG_MH_MMU_PT_BASE, mmu->hwpagetable->base.gpuaddr); kgsl_yamato_regwrite(device, REG_MH_MMU_VA_RANGE, (mmu->hwpagetable->va_base | (mmu->hwpagetable->va_range >> 16))); status = kgsl_yamato_setstate(device, KGSL_MMUFLAGS_TLBFLUSH); if (status) { kgsl_mmu_close(device); return status; } mmu->flags |= KGSL_FLAGS_STARTED; }