static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct iommu_domain *domain;
struct kgsl_iommu *iommu = mmu->priv;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
domain = mmu->hwpagetable->priv;
if (iommu->iommu_user_dev_attached) {
iommu_detach_device(domain, iommu->iommu_user_dev);
iommu->iommu_user_dev_attached = 0;
KGSL_MEM_INFO(mmu->device,
"iommu %p detached from user dev of MMU: %p\n",
domain, mmu);
}
if (iommu->iommu_priv_dev_attached) {
iommu_detach_device(domain, iommu->iommu_priv_dev);
iommu->iommu_priv_dev_attached = 0;
KGSL_MEM_INFO(mmu->device,
"iommu %p detached from priv dev of MMU: %p\n",
domain, mmu);
}
}
/*
* kgsl_detach_pagetable_iommu_domain - Detach the IOMMU unit from a
* pagetable
* @mmu - Pointer to the device mmu structure
* @priv - Flag indicating whether the private or user context is to be
* detached
*
* Detach the IOMMU unit with the domain that is contained in the
* hwpagetable of the given mmu. After detaching the IOMMU unit is not
* in use because the PTBR will not be set after a detach
* Return - void
*/
static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j;
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
iommu_pt = mmu->defaultpagetable->priv;
for (j = 0; j < iommu_unit->dev_count; j++) {
/*
* If there is a 2nd default pagetable then priv domain
* is attached with this pagetable
*/
if (mmu->priv_bank_table &&
(KGSL_IOMMU_CONTEXT_PRIV == j))
iommu_pt = mmu->priv_bank_table->priv;
if (iommu_unit->dev[j].attached) {
iommu_detach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
iommu_unit->dev[j].attached = false;
KGSL_MEM_INFO(mmu->device, "iommu %p detached "
"from user dev of MMU: %p\n",
iommu_pt->domain, mmu);
}
}
}
}
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j, ret = 0;
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
iommu_pt = mmu->defaultpagetable->priv;
for (j = 0; j < iommu_unit->dev_count; j++) {
if (mmu->priv_bank_table &&
(KGSL_IOMMU_CONTEXT_PRIV == j))
iommu_pt = mmu->priv_bank_table->priv;
if (!iommu_unit->dev[j].attached) {
ret = iommu_attach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu_unit->dev[j].attached = true;
KGSL_MEM_INFO(mmu->device,
"iommu pt %p attached to dev %p, ctx_id %d\n",
iommu_pt->domain, iommu_unit->dev[j].dev,
iommu_unit->dev[j].ctx_id);
}
}
}
done:
return ret;
}
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct iommu_domain *domain;
struct kgsl_iommu *iommu = mmu->priv;
int i, ret = 0;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
domain = mmu->hwpagetable->priv;
for (i = 0; i < iommu->dev_count; i++) {
if (iommu->dev[i].attached == 0) {
ret = iommu_attach_device(domain, iommu->dev[i].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu->dev[i].attached = 1;
KGSL_MEM_INFO(mmu->device,
"iommu %p detached from user dev of MMU: %p\n",
domain, mmu);
}
}
done:
return ret;
}
/*
* kgsl_detach_pagetable_iommu_domain - Detach the IOMMU unit from a
* pagetable
* @mmu - Pointer to the device mmu structure
* @priv - Flag indicating whether the private or user context is to be
* detached
*
* Detach the IOMMU unit with the domain that is contained in the
* hwpagetable of the given mmu. After detaching the IOMMU unit is not
* in use because the PTBR will not be set after a detach
* Return - void
*/
static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
iommu_pt = mmu->hwpagetable->priv;
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
for (j = 0; j < iommu_unit->dev_count; j++) {
if (iommu_unit->dev[j].attached) {
iommu_detach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
iommu_unit->dev[j].attached = false;
KGSL_MEM_INFO(mmu->device, "iommu %p detached "
"from user dev of MMU: %p\n",
iommu_pt->domain, mmu);
}
}
}
}
int kgsl_mmu_setstate(struct kgsl_device *device,
struct kgsl_pagetable *pagetable)
{
int status = 0;
struct kgsl_mmu *mmu = &device->mmu;
KGSL_MEM_VDBG("enter (device=%p, pagetable=%p)\n", device, pagetable);
if (mmu->flags & KGSL_FLAGS_STARTED) {
/* page table not current, then setup mmu to use new
* specified page table
*/
KGSL_MEM_INFO("from %p to %p\n", mmu->hwpagetable, pagetable);
if (mmu->hwpagetable != pagetable) {
mmu->hwpagetable = pagetable;
/* call device specific set page table */
status = kgsl_yamato_setstate(mmu->device,
KGSL_MMUFLAGS_TLBFLUSH |
KGSL_MMUFLAGS_PTUPDATE);
}
}
KGSL_MEM_VDBG("return %d\n", status);
return status;
}
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct iommu_domain *domain;
int ret = 0;
struct kgsl_iommu *iommu = mmu->priv;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
domain = mmu->hwpagetable->priv;
if (iommu->iommu_user_dev && !iommu->iommu_user_dev_attached) {
ret = iommu_attach_device(domain, iommu->iommu_user_dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n", ret);
goto done;
}
iommu->iommu_user_dev_attached = 1;
KGSL_MEM_INFO(mmu->device,
"iommu %p attached to user dev of MMU: %p\n",
domain, mmu);
}
if (iommu->iommu_priv_dev && !iommu->iommu_priv_dev_attached) {
ret = iommu_attach_device(domain, iommu->iommu_priv_dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n", ret);
iommu_detach_device(domain, iommu->iommu_user_dev);
iommu->iommu_user_dev_attached = 0;
goto done;
}
iommu->iommu_priv_dev_attached = 1;
KGSL_MEM_INFO(mmu->device,
"iommu %p attached to priv dev of MMU: %p\n",
domain, mmu);
}
done:
return ret;
}
static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct iommu_domain *domain;
struct kgsl_iommu *iommu = mmu->priv;
int i;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
domain = mmu->hwpagetable->priv;
for (i = 0; i < iommu->dev_count; i++) {
iommu_detach_device(domain, iommu->dev[i].dev);
iommu->dev[i].attached = 0;
KGSL_MEM_INFO(mmu->device,
"iommu %p detached from user dev of MMU: %p\n",
domain, mmu);
}
}
/*
* kgsl_attach_pagetable_iommu_domain - Attach the IOMMU unit to a
* pagetable, i.e set the IOMMU's PTBR to the pagetable address and
* setup other IOMMU registers for the device so that it becomes
* active
* @mmu - Pointer to the device mmu structure
* @priv - Flag indicating whether the private or user context is to be
* attached
*
* Attach the IOMMU unit with the domain that is contained in the
* hwpagetable of the given mmu.
* Return - 0 on success else error code
*/
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j, ret = 0;
/*
* Loop through all the iommu devcies under all iommu units and
* attach the domain
*/
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
iommu_pt = mmu->defaultpagetable->priv;
for (j = 0; j < iommu_unit->dev_count; j++) {
/*
* If there is a 2nd default pagetable then priv domain
* is attached to this pagetable
*/
if (mmu->priv_bank_table &&
(KGSL_IOMMU_CONTEXT_PRIV == j))
iommu_pt = mmu->priv_bank_table->priv;
if (!iommu_unit->dev[j].attached) {
ret = iommu_attach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu_unit->dev[j].attached = true;
KGSL_MEM_INFO(mmu->device,
"iommu pt %p attached to dev %p, ctx_id %d\n",
iommu_pt->domain, iommu_unit->dev[j].dev,
iommu_unit->dev[j].ctx_id);
}
}
}
done:
return ret;
}
/*
* kgsl_attach_pagetable_iommu_domain - Attach the IOMMU unit to a
* pagetable, i.e set the IOMMU's PTBR to the pagetable address and
* setup other IOMMU registers for the device so that it becomes
* active
* @mmu - Pointer to the device mmu structure
* @priv - Flag indicating whether the private or user context is to be
* attached
*
* Attach the IOMMU unit with the domain that is contained in the
* hwpagetable of the given mmu.
* Return - 0 on success else error code
*/
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j, ret = 0;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
iommu_pt = mmu->hwpagetable->priv;
/*
* Loop through all the iommu devcies under all iommu units and
* attach the domain
*/
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
for (j = 0; j < iommu_unit->dev_count; j++) {
if (!iommu_unit->dev[j].attached) {
ret = iommu_attach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu_unit->dev[j].attached = true;
KGSL_MEM_INFO(mmu->device,
"iommu pt %p attached to dev %p, ctx_id %d\n",
iommu_pt->domain, iommu_unit->dev[j].dev,
iommu_unit->dev[j].ctx_id);
}
}
}
done:
return ret;
}
int kgsl_mmu_init(struct kgsl_device *device)
{
/*
* intialize device mmu
*
* call this with the global lock held
*/
int status;
uint32_t flags;
struct kgsl_mmu *mmu = &device->mmu;
#ifdef _DEBUG
struct kgsl_mmu_debug regs;
#endif /* _DEBUG */
KGSL_MEM_VDBG("enter (device=%p)\n", device);
if (mmu->flags & KGSL_FLAGS_INITIALIZED0) {
KGSL_MEM_INFO("MMU already initialized.\n");
return 0;
}
mmu->device = device;
#ifndef CONFIG_MSM_KGSL_MMU
mmu->config = 0x00000000;
#endif
/* setup MMU and sub-client behavior */
kgsl_yamato_regwrite(device, REG_MH_MMU_CONFIG, mmu->config);
/* enable axi interrupts */
KGSL_MEM_DBG("enabling mmu interrupts mask=0x%08lx\n",
GSL_MMU_INT_MASK);
kgsl_yamato_regwrite(device, REG_MH_INTERRUPT_MASK, GSL_MMU_INT_MASK);
mmu->flags |= KGSL_FLAGS_INITIALIZED0;
/* MMU not enabled */
if ((mmu->config & 0x1) == 0) {
KGSL_MEM_VDBG("return %d\n", 0);
return 0;
}
/* idle device */
kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT);
/* make sure aligned to pagesize */
BUG_ON(mmu->mpu_base & (KGSL_PAGESIZE - 1));
BUG_ON((mmu->mpu_base + mmu->mpu_range) & (KGSL_PAGESIZE - 1));
/* define physical memory range accessible by the core */
kgsl_yamato_regwrite(device, REG_MH_MMU_MPU_BASE,
mmu->mpu_base);
kgsl_yamato_regwrite(device, REG_MH_MMU_MPU_END,
mmu->mpu_base + mmu->mpu_range);
/* enable axi interrupts */
KGSL_MEM_DBG("enabling mmu interrupts mask=0x%08lx\n",
GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT);
kgsl_yamato_regwrite(device, REG_MH_INTERRUPT_MASK,
GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT);
mmu->flags |= KGSL_FLAGS_INITIALIZED;
/* sub-client MMU lookups require address translation */
if ((mmu->config & ~0x1) > 0) {
/*make sure virtual address range is a multiple of 64Kb */
BUG_ON(mmu->va_range & ((1 << 16) - 1));
/* allocate memory used for completing r/w operations that
* cannot be mapped by the MMU
*/
flags = (KGSL_MEMFLAGS_ALIGN4K | KGSL_MEMFLAGS_CONPHYS
| KGSL_MEMFLAGS_STRICTREQUEST);
status = kgsl_sharedmem_alloc(flags, 64, &mmu->dummyspace);
if (status != 0) {
KGSL_MEM_ERR
("Unable to allocate dummy space memory.\n");
kgsl_mmu_close(device);
return status;
}
kgsl_sharedmem_set(&mmu->dummyspace, 0, 0,
mmu->dummyspace.size);
/* TRAN_ERROR needs a 32 byte (32 byte aligned) chunk of memory
* to complete transactions in case of an MMU fault. Note that
* we'll leave the bottom 32 bytes of the dummyspace for other
* purposes (e.g. use it when dummy read cycles are needed
* for other blocks */
kgsl_yamato_regwrite(device,
REG_MH_MMU_TRAN_ERROR,
mmu->dummyspace.physaddr + 32);
mmu->defaultpagetable = kgsl_mmu_createpagetableobject(mmu);
if (!mmu->defaultpagetable) {
KGSL_MEM_ERR("Failed to create global page table\n");
kgsl_mmu_close(device);
return -ENOMEM;
}
mmu->hwpagetable = mmu->defaultpagetable;
kgsl_yamato_regwrite(device, REG_MH_MMU_PT_BASE,
mmu->hwpagetable->base.gpuaddr);
kgsl_yamato_regwrite(device, REG_MH_MMU_VA_RANGE,
(mmu->hwpagetable->va_base |
(mmu->hwpagetable->va_range >> 16)));
status = kgsl_yamato_setstate(device, KGSL_MMUFLAGS_TLBFLUSH);
if (status) {
kgsl_mmu_close(device);
return status;
}
mmu->flags |= KGSL_FLAGS_STARTED;
}
