int msm_iommu_map_extra(struct iommu_domain *domain,
unsigned long start_iova,
unsigned long size,
unsigned long page_size,
int cached)
{
int ret = 0;
int i = 0;
unsigned long phy_addr = ALIGN(virt_to_phys(iommu_dummy), page_size);
unsigned long temp_iova = start_iova;
if (page_size == SZ_4K) {
struct scatterlist *sglist;
unsigned int nrpages = PFN_ALIGN(size) >> PAGE_SHIFT;
struct page *dummy_page = phys_to_page(phy_addr);
sglist = vmalloc(sizeof(*sglist) * nrpages);
if (!sglist) {
ret = -ENOMEM;
goto out;
}
sg_init_table(sglist, nrpages);
for (i = 0; i < nrpages; i++)
sg_set_page(&sglist[i], dummy_page, PAGE_SIZE, 0);
ret = iommu_map_range(domain, temp_iova, sglist, size, cached);
if (ret) {
pr_err("%s: could not map extra %lx in domain %p\n",
__func__, start_iova, domain);
}
vfree(sglist);
} else {
static int
kgsl_iommu_map(void *mmu_specific_pt,
struct kgsl_memdesc *memdesc,
unsigned int protflags,
unsigned int *tlb_flags)
{
int ret;
unsigned int iommu_virt_addr;
struct kgsl_iommu_pt *iommu_pt = mmu_specific_pt;
int size = kgsl_sg_size(memdesc->sg, memdesc->sglen);
BUG_ON(NULL == iommu_pt);
iommu_virt_addr = memdesc->gpuaddr;
ret = iommu_map_range(iommu_pt->domain, iommu_virt_addr, memdesc->sg,
size, (IOMMU_READ | IOMMU_WRITE));
if (ret) {
KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %d) "
"failed with err: %d\n", iommu_pt->domain,
iommu_virt_addr, memdesc->sg, size,
(IOMMU_READ | IOMMU_WRITE), ret);
return ret;
}
return ret;
}
static int
kgsl_iommu_map(void *mmu_specific_pt,
struct kgsl_memdesc *memdesc,
unsigned int protflags)
{
int ret;
unsigned int iommu_virt_addr;
struct iommu_domain *domain = mmu_specific_pt;
BUG_ON(NULL == domain);
iommu_virt_addr = memdesc->gpuaddr;
ret = iommu_map_range(domain, iommu_virt_addr, memdesc->sg,
memdesc->size, (IOMMU_READ | IOMMU_WRITE));
if (ret) {
KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %d) "
"failed with err: %d\n", domain,
iommu_virt_addr, memdesc->sg, memdesc->size,
0, ret);
return ret;
}
return ret;
}
static int
kgsl_iommu_map(void *mmu_specific_pt,
struct kgsl_memdesc *memdesc,
unsigned int protflags,
unsigned int *tlb_flags)
{
int ret;
unsigned int iommu_virt_addr;
struct iommu_domain *domain = mmu_specific_pt;
BUG_ON(NULL == domain);
iommu_virt_addr = memdesc->gpuaddr;
ret = iommu_map_range(domain, iommu_virt_addr, memdesc->sg,
memdesc->size, (IOMMU_READ | IOMMU_WRITE));
if (ret) {
KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %d) "
"failed with err: %d\n", domain,
iommu_virt_addr, memdesc->sg, memdesc->size,
0, ret);
return ret;
}
#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE
/*
* Flushing only required if per process pagetables are used. With
* global case, flushing will happen inside iommu_map function
*/
if (!ret)
*tlb_flags = UINT_MAX;
#endif
return ret;
}
int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
{
gfn_t gfn = slot->base_gfn;
unsigned long npages = slot->npages;
pfn_t pfn;
int i, r = 0;
struct iommu_domain *domain = kvm->arch.iommu_domain;
int flags;
/* check if iommu exists and in use */
if (!domain)
return 0;
flags = IOMMU_READ | IOMMU_WRITE;
if (kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY)
flags |= IOMMU_CACHE;
for (i = 0; i < npages; i++) {
/* check if already mapped */
if (iommu_iova_to_phys(domain, gfn_to_gpa(gfn)))
continue;
pfn = gfn_to_pfn_memslot(kvm, slot, gfn);
r = iommu_map_range(domain,
gfn_to_gpa(gfn),
pfn_to_hpa(pfn),
PAGE_SIZE, flags);
if (r) {
printk(KERN_ERR "kvm_iommu_map_address:"
"iommu failed to map pfn=%lx\n", pfn);
goto unmap_pages;
}
gfn++;
}
return 0;
unmap_pages:
kvm_iommu_put_pages(kvm, slot->base_gfn, i);
return r;
}
int msm_iommu_map_extra(struct iommu_domain *domain,
unsigned long start_iova,
phys_addr_t phy_addr,
unsigned long size,
unsigned long page_size,
int prot)
{
int ret = 0;
int i = 0;
unsigned long temp_iova = start_iova;
/* the extra "padding" should never be written to. map it
* read-only. */
prot &= ~IOMMU_WRITE;
if (msm_iommu_page_size_is_supported(page_size)) {
struct scatterlist *sglist;
unsigned int nrpages = PFN_ALIGN(size) >> PAGE_SHIFT;
struct page *dummy_page = phys_to_page(phy_addr);
sglist = vmalloc(sizeof(*sglist) * nrpages);
if (!sglist) {
ret = -ENOMEM;
goto out;
}
sg_init_table(sglist, nrpages);
for (i = 0; i < nrpages; i++)
sg_set_page(&sglist[i], dummy_page, PAGE_SIZE, 0);
ret = iommu_map_range(domain, temp_iova, sglist, size, prot);
if (ret) {
pr_err("%s: could not map extra %lx in domain %p\n",
__func__, start_iova, domain);
}
vfree(sglist);
} else {
int ion_cma_map_iommu(struct ion_buffer *buffer,
struct ion_iommu_map *data,
unsigned int domain_num,
unsigned int partition_num,
unsigned long align,
unsigned long iova_length,
unsigned long flags)
{
int ret = 0;
struct iommu_domain *domain;
unsigned long extra;
unsigned long extra_iova_addr;
struct ion_cma_buffer_info *info = buffer->priv_virt;
struct sg_table *table = info->table;
int prot = IOMMU_WRITE | IOMMU_READ;
if (!msm_use_iommu()) {
data->iova_addr = info->handle;
data->mapped_size = iova_length;
return 0;
}
data->mapped_size = iova_length;
extra = iova_length - buffer->size;
ret = msm_allocate_iova_address(domain_num, partition_num,
data->mapped_size, align,
&data->iova_addr);
if (ret)
goto out;
domain = msm_get_iommu_domain(domain_num);
if (!domain) {
ret = -EINVAL;
goto out1;
}
ret = iommu_map_range(domain, data->iova_addr, table->sgl,
buffer->size, prot);
if (ret) {
pr_err("%s: could not map %lx in domain %p\n",
__func__, data->iova_addr, domain);
goto out1;
}
extra_iova_addr = data->iova_addr + buffer->size;
if (extra) {
ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, SZ_4K,
prot);
if (ret)
goto out2;
}
return ret;
out2:
iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
out:
return ret;
}
int ion_system_contig_heap_map_iommu(struct ion_buffer *buffer,
struct ion_iommu_map *data,
unsigned int domain_num,
unsigned int partition_num,
unsigned long align,
unsigned long iova_length,
unsigned long flags)
{
int ret = 0;
struct iommu_domain *domain;
unsigned long extra;
struct scatterlist *sglist = 0;
struct page *page = 0;
int prot = IOMMU_WRITE | IOMMU_READ;
prot |= ION_IS_CACHED(flags) ? IOMMU_CACHE : 0;
if (!ION_IS_CACHED(flags))
return -EINVAL;
if (!msm_use_iommu()) {
data->iova_addr = virt_to_phys(buffer->vaddr);
return 0;
}
data->mapped_size = iova_length;
extra = iova_length - buffer->size;
data->iova_addr = msm_allocate_iova_address(domain_num, partition_num,
data->mapped_size, align);
if (!data->iova_addr) {
ret = -ENOMEM;
goto out;
}
domain = msm_get_iommu_domain(domain_num);
if (!domain) {
ret = -ENOMEM;
goto out1;
}
page = virt_to_page(buffer->vaddr);
sglist = vmalloc(sizeof(*sglist));
if (!sglist)
goto out1;
sg_init_table(sglist, 1);
sg_set_page(sglist, page, buffer->size, 0);
ret = iommu_map_range(domain, data->iova_addr, sglist,
buffer->size, prot);
if (ret) {
pr_err("%s: could not map %lx in domain %p\n",
__func__, data->iova_addr, domain);
goto out1;
}
if (extra) {
unsigned long extra_iova_addr = data->iova_addr + buffer->size;
ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, SZ_4K,
prot);
if (ret)
goto out2;
}
vfree(sglist);
return ret;
out2:
iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
out:
return ret;
}
int ion_system_heap_map_iommu(struct ion_buffer *buffer,
struct ion_iommu_map *data,
unsigned int domain_num,
unsigned int partition_num,
unsigned long align,
unsigned long iova_length,
unsigned long flags)
{
int ret = 0, i;
struct iommu_domain *domain;
unsigned long extra;
unsigned long extra_iova_addr;
struct page *page;
int npages = buffer->size >> PAGE_SHIFT;
void *vaddr = buffer->priv_virt;
struct scatterlist *sglist = 0;
int prot = IOMMU_WRITE | IOMMU_READ;
prot |= ION_IS_CACHED(flags) ? IOMMU_CACHE : 0;
if (!ION_IS_CACHED(flags))
return -EINVAL;
if (!msm_use_iommu())
return -EINVAL;
data->mapped_size = iova_length;
extra = iova_length - buffer->size;
data->iova_addr = msm_allocate_iova_address(domain_num, partition_num,
data->mapped_size, align);
if (!data->iova_addr) {
ret = -ENOMEM;
goto out;
}
domain = msm_get_iommu_domain(domain_num);
if (!domain) {
ret = -ENOMEM;
goto out1;
}
sglist = vmalloc(sizeof(*sglist) * npages);
if (!sglist) {
ret = -ENOMEM;
goto out1;
}
sg_init_table(sglist, npages);
for (i = 0; i < npages; i++) {
page = vmalloc_to_page(vaddr);
if (!page)
goto out1;
sg_set_page(&sglist[i], page, PAGE_SIZE, 0);
vaddr += PAGE_SIZE;
}
ret = iommu_map_range(domain, data->iova_addr, sglist,
buffer->size, prot);
if (ret) {
pr_err("%s: could not map %lx in domain %p\n",
__func__, data->iova_addr, domain);
goto out1;
}
extra_iova_addr = data->iova_addr + buffer->size;
if (extra) {
ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, SZ_4K,
prot);
if (ret)
goto out2;
}
vfree(sglist);
return ret;
out2:
iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
out:
return ret;
}
static int ion_cp_heap_map_iommu(struct ion_buffer *buffer,
struct ion_iommu_map *data,
unsigned int domain_num,
unsigned int partition_num,
unsigned long align,
unsigned long iova_length,
unsigned long flags)
{
struct iommu_domain *domain;
int ret = 0;
unsigned long extra;
struct scatterlist *sglist = 0;
struct ion_cp_heap *cp_heap =
container_of(buffer->heap, struct ion_cp_heap, heap);
int prot = IOMMU_WRITE | IOMMU_READ;
prot |= ION_IS_CACHED(flags) ? IOMMU_CACHE : 0;
data->mapped_size = iova_length;
if (!msm_use_iommu()) {
data->iova_addr = buffer->priv_phys;
return 0;
}
if (cp_heap->iommu_iova[domain_num]) {
/* Already mapped. */
unsigned long offset = buffer->priv_phys - cp_heap->base;
data->iova_addr = cp_heap->iommu_iova[domain_num] + offset;
return 0;
} else if (cp_heap->iommu_map_all) {
ret = iommu_map_all(domain_num, cp_heap, partition_num, prot);
if (!ret) {
unsigned long offset =
buffer->priv_phys - cp_heap->base;
data->iova_addr =
cp_heap->iommu_iova[domain_num] + offset;
cp_heap->iommu_partition[domain_num] = partition_num;
/*
clear delayed map flag so that we don't interfere
with this feature (we are already delaying).
*/
data->flags &= ~ION_IOMMU_UNMAP_DELAYED;
return 0;
} else {
cp_heap->iommu_iova[domain_num] = 0;
cp_heap->iommu_partition[domain_num] = 0;
return ret;
}
}
extra = iova_length - buffer->size;
data->iova_addr = msm_allocate_iova_address(domain_num, partition_num,
data->mapped_size, align);
if (!data->iova_addr) {
ret = -ENOMEM;
goto out;
}
domain = msm_get_iommu_domain(domain_num);
if (!domain) {
ret = -ENOMEM;
goto out1;
}
sglist = ion_cp_heap_create_sglist(buffer);
if (IS_ERR_OR_NULL(sglist)) {
ret = -ENOMEM;
goto out1;
}
ret = iommu_map_range(domain, data->iova_addr, sglist,
buffer->size, prot);
if (ret) {
pr_err("%s: could not map %lx in domain %p\n",
__func__, data->iova_addr, domain);
goto out1;
}
if (extra) {
unsigned long extra_iova_addr = data->iova_addr + buffer->size;
ret = msm_iommu_map_extra(domain, extra_iova_addr, extra,
SZ_4K, prot);
if (ret)
goto out2;
}
vfree(sglist);
return ret;
out2:
iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
if (!IS_ERR_OR_NULL(sglist))
vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
out:
return ret;
}
static int do_iommu_domain_map(struct hisi_iommu_domain *hisi_domain,struct scatterlist *sgl,
struct iommu_map_format *format, struct map_result *result)
{
int ret;
unsigned long phys_len, iova_size;
unsigned long iova_start;
struct gen_pool *pool;
struct iommu_domain *domain;
struct scatterlist *sg;
struct tile_format fmt;
/* calculate whole phys mem length */
for (phys_len = 0, sg = sgl; sg; sg = sg_next(sg)) {
phys_len += (unsigned long)ALIGN(sg->length, PAGE_SIZE);
}
/* get io virtual address size */
if (format->is_tile) {
unsigned long lines;
unsigned long body_size;
body_size = phys_len - format->header_size;
lines = body_size / (format->phys_page_line * PAGE_SIZE);
/*header need more lines virtual space*/
if ( format->header_size ){
unsigned long header_size;
header_size = ALIGN(format->header_size ,format->virt_page_line * PAGE_SIZE);
lines += header_size / (format->virt_page_line * PAGE_SIZE);
}
iova_size = lines * format->virt_page_line * PAGE_SIZE ;
} else {
iova_size = phys_len;
}
/* alloc iova */
pool = hisi_domain->iova_pool;
domain = hisi_domain->domain;
iova_start = hisi_alloc_iova(pool,iova_size,hisi_domain->range.align);
if (!iova_start) {
printk("[%s]hisi_alloc_iova alloc 0x%lx failed!\n", __func__, iova_size);
printk("[%s]dump iova pool begain--------------------------\n", __func__);
printk("iova available: 0x%x\n",(unsigned int)hisi_iommu_iova_available());
printk("alloc count: %d, free count: %d\n",
dbg_inf.alloc_iova_count, dbg_inf.free_iova_count);
printk("[%s]dump iova pool end --------------------------\n", __func__);
return -EINVAL;
}
if (0x100000000 < (iova_start + iova_size)) {
pr_err("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! "
"hisi iommu can not deal with iova 0x%lx size 0x%lx\n",
iova_start, iova_size);
}
/* do map */
if (format->is_tile) {
fmt.is_tile = format->is_tile;
fmt.phys_page_line = format->phys_page_line;
fmt.virt_page_line = format->virt_page_line;
fmt.header_size = format->header_size ;
ret = iommu_map_tile(domain, iova_start, sgl, iova_size, 0,&fmt);
} else {
ret = iommu_map_range(domain, iova_start,sgl,(size_t)iova_size,format->prot);
}
if (ret) {
printk(KERN_ERR "[%s]map failed!\n", __func__);
hisi_free_iova(pool, iova_start, iova_size);
return ret;
}else {
/* out put result */
result->iova_start = iova_start;
result->iova_size = iova_size;
}
return 0;
}