static int _load_firmware(struct kgsl_device *device, const char *fwfile,
void **data, int *len)
{
const struct firmware *fw = NULL;
int ret;
ret = request_firmware(&fw, fwfile, device->dev);
if (ret) {
KGSL_DRV_ERR(device, "request_firmware(%s) failed: %d\n",
fwfile, ret);
return ret;
}
*data = kmalloc(fw->size, GFP_KERNEL);
if (*data) {
memcpy(*data, fw->data, fw->size);
*len = fw->size;
} else
KGSL_MEM_ERR(device, "kmalloc(%d) failed\n", fw->size);
release_firmware(fw);
return (*data != NULL) ? 0 : -ENOMEM;
}
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct iommu_domain *domain;
struct kgsl_iommu *iommu = mmu->priv;
int i, ret = 0;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
domain = mmu->hwpagetable->priv;
for (i = 0; i < iommu->dev_count; i++) {
if (iommu->dev[i].attached == 0) {
ret = iommu_attach_device(domain, iommu->dev[i].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu->dev[i].attached = 1;
KGSL_MEM_INFO(mmu->device,
"iommu %p detached from user dev of MMU: %p\n",
domain, mmu);
}
}
done:
return ret;
}
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j, ret = 0;
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
iommu_pt = mmu->defaultpagetable->priv;
for (j = 0; j < iommu_unit->dev_count; j++) {
if (mmu->priv_bank_table &&
(KGSL_IOMMU_CONTEXT_PRIV == j))
iommu_pt = mmu->priv_bank_table->priv;
if (!iommu_unit->dev[j].attached) {
ret = iommu_attach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu_unit->dev[j].attached = true;
KGSL_MEM_INFO(mmu->device,
"iommu pt %p attached to dev %p, ctx_id %d\n",
iommu_pt->domain, iommu_unit->dev[j].dev,
iommu_unit->dev[j].ctx_id);
}
}
}
done:
return ret;
}
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct iommu_domain *domain;
int ret = 0;
struct kgsl_iommu *iommu = mmu->priv;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
domain = mmu->hwpagetable->priv;
if (iommu->iommu_user_dev && !iommu->iommu_user_dev_attached) {
ret = iommu_attach_device(domain, iommu->iommu_user_dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n", ret);
goto done;
}
iommu->iommu_user_dev_attached = 1;
KGSL_MEM_INFO(mmu->device,
"iommu %p attached to user dev of MMU: %p\n",
domain, mmu);
}
if (iommu->iommu_priv_dev && !iommu->iommu_priv_dev_attached) {
ret = iommu_attach_device(domain, iommu->iommu_priv_dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n", ret);
iommu_detach_device(domain, iommu->iommu_user_dev);
iommu->iommu_user_dev_attached = 0;
goto done;
}
iommu->iommu_priv_dev_attached = 1;
KGSL_MEM_INFO(mmu->device,
"iommu %p attached to priv dev of MMU: %p\n",
domain, mmu);
}
done:
return ret;
}
static long kgsl_cache_range_op(unsigned long addr, int size,
unsigned int flags)
{
#ifdef CONFIG_OUTER_CACHE
unsigned long end;
#endif
BUG_ON(addr & (KGSL_PAGESIZE - 1));
BUG_ON(size & (KGSL_PAGESIZE - 1));
if (flags & KGSL_MEMFLAGS_CACHE_FLUSH)
dmac_flush_range((const void *)addr,
(const void *)(addr + size));
else
if (flags & KGSL_MEMFLAGS_CACHE_CLEAN)
dmac_clean_range((const void *)addr,
(const void *)(addr + size));
else if (flags & KGSL_MEMFLAGS_CACHE_INV)
dmac_inv_range((const void *)addr,
(const void *)(addr + size));
#ifdef CONFIG_OUTER_CACHE
for (end = addr; end < (addr + size); end += KGSL_PAGESIZE) {
unsigned long physaddr;
if (flags & KGSL_MEMFLAGS_VMALLOC_MEM)
physaddr = page_to_phys(vmalloc_to_page((void *) end));
else
if (flags & KGSL_MEMFLAGS_HOSTADDR) {
physaddr = kgsl_virtaddr_to_physaddr(end);
if (!physaddr) {
KGSL_MEM_ERR
("Unable to find physaddr for "
"address: %x\n", (unsigned int)end);
return -EINVAL;
}
} else
return -EINVAL;
if (flags & KGSL_MEMFLAGS_CACHE_FLUSH)
outer_flush_range(physaddr, physaddr + KGSL_PAGESIZE);
else
if (flags & KGSL_MEMFLAGS_CACHE_CLEAN)
outer_clean_range(physaddr,
physaddr + KGSL_PAGESIZE);
else if (flags & KGSL_MEMFLAGS_CACHE_INV)
outer_inv_range(physaddr,
physaddr + KGSL_PAGESIZE);
}
#endif
return 0;
}
/*
* kgsl_attach_pagetable_iommu_domain - Attach the IOMMU unit to a
* pagetable, i.e set the IOMMU's PTBR to the pagetable address and
* setup other IOMMU registers for the device so that it becomes
* active
* @mmu - Pointer to the device mmu structure
* @priv - Flag indicating whether the private or user context is to be
* attached
*
* Attach the IOMMU unit with the domain that is contained in the
* hwpagetable of the given mmu.
* Return - 0 on success else error code
*/
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j, ret = 0;
/*
* Loop through all the iommu devcies under all iommu units and
* attach the domain
*/
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
iommu_pt = mmu->defaultpagetable->priv;
for (j = 0; j < iommu_unit->dev_count; j++) {
/*
* If there is a 2nd default pagetable then priv domain
* is attached to this pagetable
*/
if (mmu->priv_bank_table &&
(KGSL_IOMMU_CONTEXT_PRIV == j))
iommu_pt = mmu->priv_bank_table->priv;
if (!iommu_unit->dev[j].attached) {
ret = iommu_attach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu_unit->dev[j].attached = true;
KGSL_MEM_INFO(mmu->device,
"iommu pt %p attached to dev %p, ctx_id %d\n",
iommu_pt->domain, iommu_unit->dev[j].dev,
iommu_unit->dev[j].ctx_id);
}
}
}
done:
return ret;
}
/*
* kgsl_attach_pagetable_iommu_domain - Attach the IOMMU unit to a
* pagetable, i.e set the IOMMU's PTBR to the pagetable address and
* setup other IOMMU registers for the device so that it becomes
* active
* @mmu - Pointer to the device mmu structure
* @priv - Flag indicating whether the private or user context is to be
* attached
*
* Attach the IOMMU unit with the domain that is contained in the
* hwpagetable of the given mmu.
* Return - 0 on success else error code
*/
static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
{
struct kgsl_iommu_pt *iommu_pt;
struct kgsl_iommu *iommu = mmu->priv;
int i, j, ret = 0;
BUG_ON(mmu->hwpagetable == NULL);
BUG_ON(mmu->hwpagetable->priv == NULL);
iommu_pt = mmu->hwpagetable->priv;
/*
* Loop through all the iommu devcies under all iommu units and
* attach the domain
*/
for (i = 0; i < iommu->unit_count; i++) {
struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i];
for (j = 0; j < iommu_unit->dev_count; j++) {
if (!iommu_unit->dev[j].attached) {
ret = iommu_attach_device(iommu_pt->domain,
iommu_unit->dev[j].dev);
if (ret) {
KGSL_MEM_ERR(mmu->device,
"Failed to attach device, err %d\n",
ret);
goto done;
}
iommu_unit->dev[j].attached = true;
KGSL_MEM_INFO(mmu->device,
"iommu pt %p attached to dev %p, ctx_id %d\n",
iommu_pt->domain, iommu_unit->dev[j].dev,
iommu_unit->dev[j].ctx_id);
}
}
}
done:
return ret;
}
int kgsl_mmu_init(struct kgsl_device *device)
{
/*
* intialize device mmu
*
* call this with the global lock held
*/
int status;
uint32_t flags;
struct kgsl_mmu *mmu = &device->mmu;
#ifdef _DEBUG
struct kgsl_mmu_debug regs;
#endif /* _DEBUG */
KGSL_MEM_VDBG("enter (device=%p)\n", device);
if (mmu->flags & KGSL_FLAGS_INITIALIZED0) {
KGSL_MEM_INFO("MMU already initialized.\n");
return 0;
}
mmu->device = device;
#ifndef CONFIG_MSM_KGSL_MMU
mmu->config = 0x00000000;
#endif
/* setup MMU and sub-client behavior */
kgsl_yamato_regwrite(device, REG_MH_MMU_CONFIG, mmu->config);
/* enable axi interrupts */
KGSL_MEM_DBG("enabling mmu interrupts mask=0x%08lx\n",
GSL_MMU_INT_MASK);
kgsl_yamato_regwrite(device, REG_MH_INTERRUPT_MASK, GSL_MMU_INT_MASK);
mmu->flags |= KGSL_FLAGS_INITIALIZED0;
/* MMU not enabled */
if ((mmu->config & 0x1) == 0) {
KGSL_MEM_VDBG("return %d\n", 0);
return 0;
}
/* idle device */
kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT);
/* make sure aligned to pagesize */
BUG_ON(mmu->mpu_base & (KGSL_PAGESIZE - 1));
BUG_ON((mmu->mpu_base + mmu->mpu_range) & (KGSL_PAGESIZE - 1));
/* define physical memory range accessible by the core */
kgsl_yamato_regwrite(device, REG_MH_MMU_MPU_BASE,
mmu->mpu_base);
kgsl_yamato_regwrite(device, REG_MH_MMU_MPU_END,
mmu->mpu_base + mmu->mpu_range);
/* enable axi interrupts */
KGSL_MEM_DBG("enabling mmu interrupts mask=0x%08lx\n",
GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT);
kgsl_yamato_regwrite(device, REG_MH_INTERRUPT_MASK,
GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT);
mmu->flags |= KGSL_FLAGS_INITIALIZED;
/* sub-client MMU lookups require address translation */
if ((mmu->config & ~0x1) > 0) {
/*make sure virtual address range is a multiple of 64Kb */
BUG_ON(mmu->va_range & ((1 << 16) - 1));
/* allocate memory used for completing r/w operations that
* cannot be mapped by the MMU
*/
flags = (KGSL_MEMFLAGS_ALIGN4K | KGSL_MEMFLAGS_CONPHYS
| KGSL_MEMFLAGS_STRICTREQUEST);
status = kgsl_sharedmem_alloc(flags, 64, &mmu->dummyspace);
if (status != 0) {
KGSL_MEM_ERR
("Unable to allocate dummy space memory.\n");
kgsl_mmu_close(device);
return status;
}
kgsl_sharedmem_set(&mmu->dummyspace, 0, 0,
mmu->dummyspace.size);
/* TRAN_ERROR needs a 32 byte (32 byte aligned) chunk of memory
* to complete transactions in case of an MMU fault. Note that
* we'll leave the bottom 32 bytes of the dummyspace for other
* purposes (e.g. use it when dummy read cycles are needed
* for other blocks */
kgsl_yamato_regwrite(device,
REG_MH_MMU_TRAN_ERROR,
mmu->dummyspace.physaddr + 32);
mmu->defaultpagetable = kgsl_mmu_createpagetableobject(mmu);
if (!mmu->defaultpagetable) {
KGSL_MEM_ERR("Failed to create global page table\n");
kgsl_mmu_close(device);
return -ENOMEM;
}
mmu->hwpagetable = mmu->defaultpagetable;
kgsl_yamato_regwrite(device, REG_MH_MMU_PT_BASE,
mmu->hwpagetable->base.gpuaddr);
kgsl_yamato_regwrite(device, REG_MH_MMU_VA_RANGE,
(mmu->hwpagetable->va_base |
(mmu->hwpagetable->va_range >> 16)));
status = kgsl_yamato_setstate(device, KGSL_MMUFLAGS_TLBFLUSH);
if (status) {
kgsl_mmu_close(device);
return status;
}
mmu->flags |= KGSL_FLAGS_STARTED;
}
struct kgsl_pagetable *kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu)
{
int status = 0;
struct kgsl_pagetable *pagetable = NULL;
uint32_t flags;
KGSL_MEM_VDBG("enter (mmu=%p)\n", mmu);
pagetable = kzalloc(sizeof(struct kgsl_pagetable), GFP_KERNEL);
if (pagetable == NULL) {
KGSL_MEM_ERR("Unable to allocate pagetable object.\n");
return NULL;
}
pagetable->mmu = mmu;
pagetable->va_base = mmu->va_base;
pagetable->va_range = mmu->va_range;
pagetable->last_superpte = 0;
pagetable->max_entries = (mmu->va_range >> KGSL_PAGESIZE_SHIFT)
+ GSL_PT_EXTRA_ENTRIES;
pagetable->pool = gen_pool_create(KGSL_PAGESIZE_SHIFT, -1);
if (pagetable->pool == NULL) {
KGSL_MEM_ERR("Unable to allocate virtualaddr pool.\n");
goto err_gen_pool_create;
}
if (gen_pool_add(pagetable->pool, pagetable->va_base,
pagetable->va_range, -1)) {
KGSL_MEM_ERR("gen_pool_create failed for pagetable %p\n",
pagetable);
goto err_gen_pool_add;
}
/* allocate page table memory */
flags = (KGSL_MEMFLAGS_ALIGN4K | KGSL_MEMFLAGS_CONPHYS
| KGSL_MEMFLAGS_STRICTREQUEST);
status = kgsl_sharedmem_alloc(flags,
pagetable->max_entries * GSL_PTE_SIZE,
&pagetable->base);
if (status) {
KGSL_MEM_ERR("cannot alloc page tables\n");
goto err_kgsl_sharedmem_alloc;
}
/* reset page table entries
* -- all pte's are marked as not dirty initially
*/
kgsl_sharedmem_set(&pagetable->base, 0, 0, pagetable->base.size);
pagetable->base.gpuaddr = pagetable->base.physaddr;
KGSL_MEM_VDBG("return %p\n", pagetable);
return pagetable;
err_kgsl_sharedmem_alloc:
err_gen_pool_add:
gen_pool_destroy(pagetable->pool);
err_gen_pool_create:
kfree(pagetable);
return NULL;
}
