/* * kgsl_iommu_create_pagetable - Create a IOMMU pagetable * * Allocate memory to hold a pagetable and allocate the IOMMU * domain which is the actual IOMMU pagetable * Return - void */ void *kgsl_iommu_create_pagetable(void) { struct kgsl_iommu_pt *iommu_pt; iommu_pt = kzalloc(sizeof(struct kgsl_iommu_pt), GFP_KERNEL); if (!iommu_pt) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu_pt)); return NULL; } /* L2 redirect is not stable on IOMMU v2 */ if (msm_soc_version_supports_iommu_v1()) iommu_pt->domain = iommu_domain_alloc(&platform_bus_type, MSM_IOMMU_DOMAIN_PT_CACHEABLE); else iommu_pt->domain = iommu_domain_alloc(&platform_bus_type, 0); if (!iommu_pt->domain) { KGSL_CORE_ERR("Failed to create iommu domain\n"); kfree(iommu_pt); return NULL; } else { iommu_set_fault_handler(iommu_pt->domain, kgsl_iommu_fault_handler); } return iommu_pt; }
static int _get_iommu_ctxs(struct kgsl_iommu *iommu, struct kgsl_device *device, struct kgsl_device_iommu_data *data) { int i; for (i = 0; i < data->iommu_ctx_count; i++) { if (iommu->dev_count >= KGSL_IOMMU_MAX_DEV) { KGSL_CORE_ERR("Tried to attach too many IOMMU " "devices\n"); return -ENOMEM; } if (!data->iommu_ctx_names[i]) continue; iommu->dev[iommu->dev_count].dev = msm_iommu_get_ctx(data->iommu_ctx_names[i]); if (iommu->dev[iommu->dev_count].dev == NULL) { KGSL_CORE_ERR("Failed to iommu dev handle for " "device %s\n", data->iommu_ctx_names[i]); return -EINVAL; } iommu->dev_count++; } return 0; }
/* Push a new buffer object onto the list */ static void push_object(int type, struct kgsl_process_private *process, uint64_t gpuaddr, uint64_t dwords) { int index; struct kgsl_mem_entry *entry; if (process == NULL) return; /* * Sometimes IBs can be reused in the same dump. Because we parse from * oldest to newest, if we come across an IB that has already been used, * assume that it has been reused and update the list with the newest * size. */ for (index = 0; index < objbufptr; index++) { if (objbuf[index].gpuaddr == gpuaddr && objbuf[index].entry->priv == process) { objbuf[index].size = max_t(uint64_t, objbuf[index].size, dwords << 2); return; } } if (objbufptr == SNAPSHOT_OBJ_BUFSIZE) { KGSL_CORE_ERR("snapshot: too many snapshot objects\n"); return; } entry = kgsl_sharedmem_find(process, gpuaddr); if (entry == NULL) { KGSL_CORE_ERR("snapshot: Can't find entry for 0x%016llX\n", gpuaddr); return; } if (!kgsl_gpuaddr_in_memdesc(&entry->memdesc, gpuaddr, dwords << 2)) { KGSL_CORE_ERR("snapshot: Mem entry 0x%016llX is too small\n", gpuaddr); kgsl_mem_entry_put(entry); return; } /* Put it on the list of things to parse */ objbuf[objbufptr].type = type; objbuf[objbufptr].gpuaddr = gpuaddr; objbuf[objbufptr].size = dwords << 2; objbuf[objbufptr++].entry = entry; }
static int kgsl_iommu_init(struct kgsl_mmu *mmu) { /* * intialize device mmu * * call this with the global lock held */ int status = 0; struct kgsl_iommu *iommu; iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL); if (!iommu) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); return -ENOMEM; } iommu->asids = kzalloc(BITS_TO_LONGS(KGSL_IOMMU_MAX_ASIDS) * sizeof(unsigned long), GFP_KERNEL); if (!iommu->asids) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); status = -ENOMEM; goto done; } mmu->priv = iommu; status = kgsl_get_iommu_ctxt(mmu); if (status) goto done; status = kgsl_set_register_map(mmu); if (status) goto done; /* A nop is required in an indirect buffer when switching * pagetables in-stream */ kgsl_sharedmem_writel(&mmu->setstate_memory, KGSL_IOMMU_SETSTATE_NOP_OFFSET, cp_nop_packet(1)); dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n", __func__); done: if (status) { kfree(iommu->asids); kfree(iommu); mmu->priv = NULL; } return status; }
/* * kgsl_set_register_map - Map the IOMMU regsiters in the memory descriptors * of the respective iommu units * @mmu - Pointer to mmu structure * * Return - 0 on success else error code */ static int kgsl_set_register_map(struct kgsl_mmu *mmu) { struct platform_device *pdev = container_of(mmu->device->parentdev, struct platform_device, dev); struct kgsl_device_platform_data *pdata_dev = pdev->dev.platform_data; struct kgsl_iommu *iommu = mmu->device->mmu.priv; struct kgsl_iommu_unit *iommu_unit; int i = 0, ret = 0; for (; i < pdata_dev->iommu_count; i++) { struct kgsl_device_iommu_data data = pdata_dev->iommu_data[i]; iommu_unit = &iommu->iommu_units[i]; /* set up the IOMMU register map for the given IOMMU unit */ if (!data.physstart || !data.physend) { KGSL_CORE_ERR("The register range for IOMMU unit not" " specified\n"); ret = -EINVAL; goto err; } iommu_unit->reg_map.hostptr = ioremap(data.physstart, data.physend - data.physstart + 1); if (!iommu_unit->reg_map.hostptr) { KGSL_CORE_ERR("Failed to map SMMU register address " "space from %x to %x\n", data.physstart, data.physend - data.physstart + 1); ret = -ENOMEM; i--; goto err; } iommu_unit->reg_map.size = data.physend - data.physstart + 1; iommu_unit->reg_map.physaddr = data.physstart; ret = memdesc_sg_phys(&iommu_unit->reg_map, data.physstart, iommu_unit->reg_map.size); if (ret) goto err; } iommu->unit_count = pdata_dev->iommu_count; return ret; err: /* Unmap any mapped IOMMU regions */ for (; i >= 0; i--) { iommu_unit = &iommu->iommu_units[i]; iounmap(iommu_unit->reg_map.hostptr); iommu_unit->reg_map.size = 0; iommu_unit->reg_map.physaddr = 0; } return ret; }
/* * _get_iommu_ctxs - Get device pointer to IOMMU contexts * @mmu - Pointer to mmu device * data - Pointer to the platform data containing information about * iommu devices for one iommu unit * unit_id - The IOMMU unit number. This is not a specific ID but just * a serial number. The serial numbers are treated as ID's of the * IOMMU units * * Return - 0 on success else error code */ static int _get_iommu_ctxs(struct kgsl_mmu *mmu, struct kgsl_device_iommu_data *data, unsigned int unit_id) { struct kgsl_iommu *iommu = mmu->priv; struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[unit_id]; int i, j; int found_ctx; for (j = 0; j < KGSL_IOMMU_MAX_DEVS_PER_UNIT; j++) { found_ctx = 0; for (i = 0; i < data->iommu_ctx_count; i++) { if (j == data->iommu_ctxs[i].ctx_id) { found_ctx = 1; break; } } if (!found_ctx) break; if (!data->iommu_ctxs[i].iommu_ctx_name) { KGSL_CORE_ERR("Context name invalid\n"); return -EINVAL; } iommu_unit->dev[iommu_unit->dev_count].dev = msm_iommu_get_ctx(data->iommu_ctxs[i].iommu_ctx_name); if (iommu_unit->dev[iommu_unit->dev_count].dev == NULL) { KGSL_CORE_ERR("Failed to get iommu dev handle for " "device %s\n", data->iommu_ctxs[i].iommu_ctx_name); return -EINVAL; } iommu_unit->dev[iommu_unit->dev_count].ctx_id = data->iommu_ctxs[i].ctx_id; iommu_unit->dev[iommu_unit->dev_count].kgsldev = mmu->device; KGSL_DRV_INFO(mmu->device, "Obtained dev handle %p for iommu context %s\n", iommu_unit->dev[iommu_unit->dev_count].dev, data->iommu_ctxs[i].iommu_ctx_name); iommu_unit->dev_count++; } if (!j) { KGSL_CORE_ERR("No ctxts initialized, user ctxt absent\n "); return -EINVAL; } return 0; }
static int kgsl_iommu_unmap(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int *tlb_flags) { int ret; unsigned int range = kgsl_sg_size(memdesc->sg, memdesc->sglen); struct kgsl_iommu_pt *iommu_pt = mmu_specific_pt; unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK; if (range == 0 || gpuaddr == 0) return 0; ret = iommu_unmap_range(iommu_pt->domain, gpuaddr, range); if (ret) KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed " "with err: %d\n", iommu_pt->domain, gpuaddr, range, ret); #ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE if (!ret) *tlb_flags = UINT_MAX; #endif return 0; }
static int kgsl_iommu_init(struct kgsl_mmu *mmu) { int status = 0; struct kgsl_iommu *iommu; iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL); if (!iommu) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); return -ENOMEM; } mmu->priv = iommu; status = kgsl_get_iommu_ctxt(mmu); if (status) goto done; status = kgsl_set_register_map(mmu); if (status) goto done; kgsl_sharedmem_writel(&mmu->setstate_memory, KGSL_IOMMU_SETSTATE_NOP_OFFSET, cp_nop_packet(1)); dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n", __func__); done: if (status) { kfree(iommu); mmu->priv = NULL; } return status; }
static int kgsl_iommu_fault_handler(struct iommu_domain *domain, struct device *dev, unsigned long addr, int flags) { struct kgsl_iommu_unit *iommu_unit = get_iommu_unit(dev); struct kgsl_iommu_device *iommu_dev = get_iommu_device(iommu_unit, dev); unsigned int ptbase, fsr; if (!iommu_dev) { KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev); return -ENOSYS; } ptbase = KGSL_IOMMU_GET_IOMMU_REG(iommu_unit->reg_map.hostptr, iommu_dev->ctx_id, TTBR0); fsr = KGSL_IOMMU_GET_IOMMU_REG(iommu_unit->reg_map.hostptr, iommu_dev->ctx_id, FSR); KGSL_MEM_CRIT(iommu_dev->kgsldev, "GPU PAGE FAULT: addr = %lX pid = %d\n", addr, kgsl_mmu_get_ptname_from_ptbase(ptbase)); KGSL_MEM_CRIT(iommu_dev->kgsldev, "context = %d FSR = %X\n", iommu_dev->ctx_id, fsr); trace_kgsl_mmu_pagefault(iommu_dev->kgsldev, addr, kgsl_mmu_get_ptname_from_ptbase(ptbase), 0); return 0; }
static int kgsl_iommu_init(struct kgsl_mmu *mmu) { /* * intialize device mmu * * call this with the global lock held */ int status = 0; struct kgsl_iommu *iommu; iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL); if (!iommu) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); return -ENOMEM; } mmu->priv = iommu; status = kgsl_get_iommu_ctxt(mmu); if (status) goto done; dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n", __func__); done: if (status) { kfree(iommu); mmu->priv = NULL; } return status; }
static int kgsl_iommu_unmap(void *mmu_specific_pt, struct kgsl_memdesc *memdesc) { int ret; unsigned int range = memdesc->size; struct iommu_domain *domain = (struct iommu_domain *) mmu_specific_pt; /* All GPU addresses as assigned are page aligned, but some functions purturb the gpuaddr with an offset, so apply the mask here to make sure we have the right address */ unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK; if (range == 0 || gpuaddr == 0) return 0; ret = iommu_unmap_range(domain, gpuaddr, range); if (ret) KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed " "with err: %d\n", domain, gpuaddr, range, ret); return 0; }
static int kgsl_iommu_map(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int protflags, unsigned int *tlb_flags) { int ret; unsigned int iommu_virt_addr; struct kgsl_iommu_pt *iommu_pt = mmu_specific_pt; int size = kgsl_sg_size(memdesc->sg, memdesc->sglen); BUG_ON(NULL == iommu_pt); iommu_virt_addr = memdesc->gpuaddr; ret = iommu_map_range(iommu_pt->domain, iommu_virt_addr, memdesc->sg, size, (IOMMU_READ | IOMMU_WRITE)); if (ret) { KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %d) " "failed with err: %d\n", iommu_pt->domain, iommu_virt_addr, memdesc->sg, size, (IOMMU_READ | IOMMU_WRITE), ret); return ret; } return ret; }
static int kgsl_iommu_init(struct kgsl_device *device) { /* * intialize device mmu * * call this with the global lock held */ int status = 0; struct kgsl_mmu *mmu = &device->mmu; struct kgsl_iommu *iommu; mmu->device = device; iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL); if (!iommu) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); return -ENOMEM; } iommu->iommu_priv_dev_attached = 0; iommu->iommu_user_dev_attached = 0; status = kgsl_get_iommu_ctxt(iommu, device); if (status) { kfree(iommu); iommu = NULL; } mmu->priv = iommu; dev_info(device->dev, "|%s| MMU type set for device is IOMMU\n", __func__); return status; }
/* * kgsl_get_iommu_ctxt - Get device pointer to IOMMU contexts * @mmu - Pointer to mmu device * * Get the device pointers for the IOMMU user and priv contexts of the * kgsl device * Return - 0 on success else error code */ static int kgsl_get_iommu_ctxt(struct kgsl_mmu *mmu) { struct platform_device *pdev = container_of(mmu->device->parentdev, struct platform_device, dev); struct kgsl_device_platform_data *pdata_dev = pdev->dev.platform_data; struct kgsl_iommu *iommu = mmu->device->mmu.priv; int i, ret = 0; /* Go through the IOMMU data and get all the context devices */ if (KGSL_IOMMU_MAX_UNITS < pdata_dev->iommu_count) { KGSL_CORE_ERR("Too many IOMMU units defined\n"); ret = -EINVAL; goto done; } for (i = 0; i < pdata_dev->iommu_count; i++) { ret = _get_iommu_ctxs(mmu, &pdata_dev->iommu_data[i], i); if (ret) break; } iommu->unit_count = pdata_dev->iommu_count; done: return ret; }
static int kgsl_iommu_map(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int protflags, unsigned int *tlb_flags) { int ret; unsigned int iommu_virt_addr; struct iommu_domain *domain = mmu_specific_pt; BUG_ON(NULL == domain); iommu_virt_addr = memdesc->gpuaddr; ret = iommu_map_range(domain, iommu_virt_addr, memdesc->sg, memdesc->size, MSM_IOMMU_ATTR_NONCACHED); if (ret) { KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %d) " "failed with err: %d\n", domain, iommu_virt_addr, memdesc->sg, memdesc->size, MSM_IOMMU_ATTR_NONCACHED, ret); return ret; } #ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE /* * Flushing only required if per process pagetables are used. With * global case, flushing will happen inside iommu_map function */ if (!ret) *tlb_flags = UINT_MAX; #endif return ret; }
/* * kgsl_iommu_get_reg_map_desc - Returns an array of pointers that contain * the address of memory descriptors which map the IOMMU registers * @mmu - Pointer to mmu structure * @reg_map_desc - Out parameter in which the address of the array containing * pointers to register map descriptors is returned. The caller is supposed * to free this array * * Return - The number of iommu units which is also the number of register * mapped descriptor arrays which the out parameter will have */ static int kgsl_iommu_get_reg_map_desc(struct kgsl_mmu *mmu, void **reg_map_desc) { struct kgsl_iommu *iommu = mmu->priv; void **reg_desc_ptr; int i; /* * Alocate array of pointers that will hold address of the register map * descriptors */ reg_desc_ptr = kmalloc(iommu->unit_count * sizeof(struct kgsl_memdesc *), GFP_KERNEL); if (!reg_desc_ptr) { KGSL_CORE_ERR("Failed to kmalloc(%d)\n", iommu->unit_count * sizeof(struct kgsl_memdesc *)); return -ENOMEM; } for (i = 0; i < iommu->unit_count; i++) reg_desc_ptr[i] = &(iommu->iommu_units[i].reg_map); *reg_map_desc = reg_desc_ptr; return i; }
static int kgsl_iommu_unmap(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int *tlb_flags) { int ret; unsigned int range = kgsl_sg_size(memdesc->sg, memdesc->sglen); struct kgsl_iommu_pt *iommu_pt = mmu_specific_pt; /* All GPU addresses as assigned are page aligned, but some functions purturb the gpuaddr with an offset, so apply the mask here to make sure we have the right address */ unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK; if (range == 0 || gpuaddr == 0) return 0; ret = iommu_unmap_range(iommu_pt->domain, gpuaddr, range); if (ret) KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed " "with err: %d\n", iommu_pt->domain, gpuaddr, range, ret); #ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE /* * Flushing only required if per process pagetables are used. With * global case, flushing will happen inside iommu_map function */ if (!ret && msm_soc_version_supports_iommu_v1()) *tlb_flags = UINT_MAX; #endif return 0; }
static int kgsl_iommu_map(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int protflags) { int ret; unsigned int iommu_virt_addr; struct iommu_domain *domain = mmu_specific_pt; BUG_ON(NULL == domain); iommu_virt_addr = memdesc->gpuaddr; ret = iommu_map_range(domain, iommu_virt_addr, memdesc->sg, memdesc->size, 0); if (ret) { KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %d) " "failed with err: %d\n", domain, iommu_virt_addr, memdesc->sg, memdesc->size, 0, ret); return ret; } return ret; }
void *kgsl_iommu_create_pagetable(void) { struct iommu_domain *domain = iommu_domain_alloc(0); if (!domain) KGSL_CORE_ERR("Failed to create iommu domain\n"); return domain; }
/* * _get_iommu_ctxs - Get device pointer to IOMMU contexts * @mmu - Pointer to mmu device * data - Pointer to the platform data containing information about * iommu devices for one iommu unit * unit_id - The IOMMU unit number. This is not a specific ID but just * a serial number. The serial numbers are treated as ID's of the * IOMMU units * * Return - 0 on success else error code */ static int _get_iommu_ctxs(struct kgsl_mmu *mmu, struct kgsl_device_iommu_data *data, unsigned int unit_id) { struct kgsl_iommu *iommu = mmu->priv; struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[unit_id]; int i; if (data->iommu_ctx_count > KGSL_IOMMU_MAX_DEVS_PER_UNIT) { KGSL_CORE_ERR("Too many iommu devices defined for an " "IOMMU unit\n"); return -EINVAL; } for (i = 0; i < data->iommu_ctx_count; i++) { if (!data->iommu_ctxs[i].iommu_ctx_name) continue; iommu_unit->dev[iommu_unit->dev_count].dev = msm_iommu_get_ctx(data->iommu_ctxs[i].iommu_ctx_name); if (iommu_unit->dev[iommu_unit->dev_count].dev == NULL) { KGSL_CORE_ERR("Failed to get iommu dev handle for " "device %s\n", data->iommu_ctxs[i].iommu_ctx_name); return -EINVAL; } if (KGSL_IOMMU_CONTEXT_USER != data->iommu_ctxs[i].ctx_id && KGSL_IOMMU_CONTEXT_PRIV != data->iommu_ctxs[i].ctx_id) { KGSL_CORE_ERR("Invalid context ID defined: %d\n", data->iommu_ctxs[i].ctx_id); return -EINVAL; } iommu_unit->dev[iommu_unit->dev_count].ctx_id = data->iommu_ctxs[i].ctx_id; iommu_unit->dev[iommu_unit->dev_count].kgsldev = mmu->device; KGSL_DRV_INFO(mmu->device, "Obtained dev handle %p for iommu context %s\n", iommu_unit->dev[iommu_unit->dev_count].dev, data->iommu_ctxs[i].iommu_ctx_name); iommu_unit->dev_count++; } return 0; }
static int kgsl_iommu_map(void *mmu_specific_pt, struct kgsl_memdesc *memdesc, unsigned int protflags) { int ret = 0; unsigned int physaddr; unsigned int iommu_virt_addr; unsigned int offset = 0; int map_order; struct iommu_domain *domain = (struct iommu_domain *) mmu_specific_pt; BUG_ON(NULL == domain); map_order = get_order(SZ_4K); for (iommu_virt_addr = memdesc->gpuaddr; iommu_virt_addr < (memdesc->gpuaddr + memdesc->size); iommu_virt_addr += SZ_4K, offset += PAGE_SIZE) { physaddr = memdesc->ops->physaddr(memdesc, offset); if (!physaddr) { KGSL_CORE_ERR("Failed to convert %x address to " "physical\n", (unsigned int)memdesc->hostptr + offset); kgsl_iommu_unmap(mmu_specific_pt, memdesc); return -EFAULT; } ret = iommu_map(domain, iommu_virt_addr, physaddr, map_order, MSM_IOMMU_ATTR_NONCACHED); if (ret) { KGSL_CORE_ERR("iommu_map(%p, %x, %x, %d, %d) " "failed with err: %d\n", domain, iommu_virt_addr, physaddr, map_order, MSM_IOMMU_ATTR_NONCACHED, ret); kgsl_iommu_unmap(mmu_specific_pt, memdesc); return ret; } } return ret; }
static inline struct kgsl_mem_entry * kgsl_mem_entry_create(void) { struct kgsl_mem_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(*entry)); else kref_init(&entry->refcount); return entry; }
static int kgsl_iommu_pt_equal(struct kgsl_pagetable *pt, unsigned int pt_base) { struct iommu_domain *domain; if (!pt) { KGSL_CORE_ERR("pt is NULL\n"); return 0; } domain = pt->priv; return pt && pt_base && ((unsigned int)domain == pt_base); }
static int kgsl_iommu_init(struct kgsl_mmu *mmu) { /* * intialize device mmu * * call this with the global lock held */ int status = 0; struct kgsl_iommu *iommu; iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL); if (!iommu) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu)); return -ENOMEM; } mmu->priv = iommu; status = kgsl_get_iommu_ctxt(mmu); if (status) goto done; status = kgsl_set_register_map(mmu); if (status) goto done; iommu->iommu_reg_list = kgsl_iommuv1_reg; iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1; if (msm_soc_version_supports_iommu_v1()) { iommu->iommu_reg_list = kgsl_iommuv1_reg; iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1; } else { iommu->iommu_reg_list = kgsl_iommuv2_reg; iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V2; } /* A nop is required in an indirect buffer when switching * pagetables in-stream */ kgsl_sharedmem_writel(&mmu->setstate_memory, KGSL_IOMMU_SETSTATE_NOP_OFFSET, cp_nop_packet(1)); dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n", __func__); done: if (status) { kfree(iommu); mmu->priv = NULL; } return status; }
/* * kgsl_iommu_create_pagetable - Create a IOMMU pagetable * * Allocate memory to hold a pagetable and allocate the IOMMU * domain which is the actual IOMMU pagetable * Return - void */ void *kgsl_iommu_create_pagetable(void) { struct kgsl_iommu_pt *iommu_pt; iommu_pt = kzalloc(sizeof(struct kgsl_iommu_pt), GFP_KERNEL); if (!iommu_pt) { KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(struct kgsl_iommu_pt)); return NULL; } iommu_pt->domain = iommu_domain_alloc(&platform_bus_type, MSM_IOMMU_DOMAIN_PT_CACHEABLE); if (!iommu_pt->domain) { KGSL_CORE_ERR("Failed to create iommu domain\n"); kfree(iommu_pt); return NULL; } else { iommu_set_fault_handler(iommu_pt->domain, kgsl_iommu_fault_handler); } return iommu_pt; }
static int kgsl_iommu_fault_handler(struct iommu_domain *domain, struct device *dev, unsigned long addr, int flags) { struct kgsl_iommu_unit *iommu_unit = get_iommu_unit(dev); struct kgsl_iommu_device *iommu_dev = get_iommu_device(iommu_unit, dev); unsigned int ptbase, fsr; static unsigned long last_pagefault_jiffies; static int last_pid; int current_pid; unsigned long wait_time_jiff = 0; if (!iommu_dev) { KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev); return -ENOSYS; } wait_time_jiff = last_pagefault_jiffies + msecs_to_jiffies(500); last_pagefault_jiffies = jiffies; ptbase = KGSL_IOMMU_GET_IOMMU_REG(iommu_unit->reg_map.hostptr, iommu_dev->ctx_id, TTBR0); current_pid = kgsl_mmu_get_ptname_from_ptbase(ptbase); /* * Lots of pagefaults in the same process that range over a short amount * of time are likely to be part of the same problem, so try to throttle * the number of messages being printed. */ if ((last_pid != current_pid) || (time_after(jiffies, wait_time_jiff)) ) { fsr = KGSL_IOMMU_GET_IOMMU_REG(iommu_unit->reg_map.hostptr, iommu_dev->ctx_id, FSR); KGSL_MEM_CRIT(iommu_dev->kgsldev, "GPU PAGE FAULT: addr = %lX pid = %d\n", addr, kgsl_mmu_get_ptname_from_ptbase(ptbase)); KGSL_MEM_CRIT(iommu_dev->kgsldev, "context = %d FSR = %X\n", iommu_dev->ctx_id, fsr); last_pid = current_pid; } trace_kgsl_mmu_pagefault(iommu_dev->kgsldev, addr, kgsl_mmu_get_ptname_from_ptbase(ptbase), 0); return 0; }
static int kgsl_iommu_fault_handler(struct iommu_domain *domain, struct device *dev, unsigned long addr, int flags) { struct kgsl_iommu_unit *iommu_unit = get_iommu_unit(dev); struct kgsl_iommu_device *iommu_dev = get_iommu_device(iommu_unit, dev); unsigned int ptbase, fsr; static unsigned long last_pagefault_jiffies; static int last_pid; int current_pid; unsigned long wait_time_jiff = 0; if (!iommu_dev) { KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev); return -ENOSYS; } wait_time_jiff = last_pagefault_jiffies + msecs_to_jiffies(500); last_pagefault_jiffies = jiffies; ptbase = KGSL_IOMMU_GET_IOMMU_REG(iommu_unit->reg_map.hostptr, iommu_dev->ctx_id, TTBR0); current_pid = kgsl_mmu_get_ptname_from_ptbase(ptbase); if ((last_pid != current_pid) || (time_after(jiffies, wait_time_jiff)) ) { fsr = KGSL_IOMMU_GET_IOMMU_REG(iommu_unit->reg_map.hostptr, iommu_dev->ctx_id, FSR); KGSL_MEM_CRIT(iommu_dev->kgsldev, "GPU PAGE FAULT: addr = %lX pid = %d\n", addr, kgsl_mmu_get_ptname_from_ptbase(ptbase)); KGSL_MEM_CRIT(iommu_dev->kgsldev, "context = %d FSR = %X\n", iommu_dev->ctx_id, fsr); last_pid = current_pid; } trace_kgsl_mmu_pagefault(iommu_dev->kgsldev, addr, kgsl_mmu_get_ptname_from_ptbase(ptbase), 0); return 0; }
static int kgsl_iommu_get_reg_map_desc(struct kgsl_mmu *mmu, void **reg_map_desc) { struct kgsl_iommu *iommu = mmu->priv; void **reg_desc_ptr; int i; reg_desc_ptr = kmalloc(iommu->unit_count * sizeof(struct kgsl_memdesc *), GFP_KERNEL); if (!reg_desc_ptr) { KGSL_CORE_ERR("Failed to kmalloc(%d)\n", iommu->unit_count * sizeof(struct kgsl_memdesc *)); return -ENOMEM; } for (i = 0; i < iommu->unit_count; i++) reg_desc_ptr[i] = &(iommu->iommu_units[i].reg_map); *reg_map_desc = reg_desc_ptr; return i; }
static int kgsl_get_iommu_ctxt(struct kgsl_iommu *iommu, struct kgsl_device *device) { int status = 0; struct platform_device *pdev = container_of(device->parentdev, struct platform_device, dev); struct kgsl_device_platform_data *pdata_dev = pdev->dev.platform_data; if (pdata_dev->iommu_user_ctx_name) iommu->iommu_user_dev = msm_iommu_get_ctx( pdata_dev->iommu_user_ctx_name); if (pdata_dev->iommu_priv_ctx_name) iommu->iommu_priv_dev = msm_iommu_get_ctx( pdata_dev->iommu_priv_ctx_name); if (!iommu->iommu_user_dev) { KGSL_CORE_ERR("Failed to get user iommu dev handle for " "device %s\n", pdata_dev->iommu_user_ctx_name); status = -EINVAL; } return status; }
static int kgsl_iommu_fault_handler(struct iommu_domain *domain, struct device *dev, unsigned long addr, int flags) { int ret = 0; struct kgsl_mmu *mmu; struct kgsl_iommu *iommu; struct kgsl_iommu_unit *iommu_unit; struct kgsl_iommu_device *iommu_dev; unsigned int ptbase, fsr; ret = get_iommu_unit(dev, &mmu, &iommu_unit); if (ret) goto done; iommu_dev = get_iommu_device(iommu_unit, dev); if (!iommu_dev) { KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev); ret = -ENOSYS; goto done; } iommu = mmu->priv; ptbase = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, iommu_dev->ctx_id, TTBR0); fsr = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, iommu_dev->ctx_id, FSR); KGSL_MEM_CRIT(iommu_dev->kgsldev, "GPU PAGE FAULT: addr = %lX pid = %d\n", addr, kgsl_mmu_get_ptname_from_ptbase(mmu, ptbase)); KGSL_MEM_CRIT(iommu_dev->kgsldev, "context = %d FSR = %X\n", iommu_dev->ctx_id, fsr); trace_kgsl_mmu_pagefault(iommu_dev->kgsldev, addr, kgsl_mmu_get_ptname_from_ptbase(mmu, ptbase), 0); done: return ret; }