/* * Initialise the coherent pool for atomic allocations. */ static int __init coherent_init(void) { pgprot_t prot = pgprot_dmacoherent(pgprot_kernel); size_t size = coherent_pool_size; struct page *page; void *ptr; if (cma_available) ptr = __alloc_from_contiguous(NULL, size, prot, &page); else ptr = __alloc_remap_buffer(NULL, size, GFP_KERNEL | GFP_DMA, prot, &page, NULL); if (ptr) { coherent_head.vm_start = (unsigned long) ptr; coherent_head.vm_end = (unsigned long) ptr + size; coherent_head.vm_pages = page; printk(KERN_INFO "DMA: preallocated %u KiB pool for atomic coherent allocations\n", (unsigned)size / 1024); return 0; } printk(KERN_ERR "DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", (unsigned)size / 1024); return -ENOMEM; }
static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, bool coherent) { if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) return pgprot_writecombine(prot); else if (!coherent) return pgprot_dmacoherent(prot); return prot; }
int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot); if (! dma_mmap_from_coherent(dev, vma, cpu_addr, dma_addr, size)) return 0; return dma_mmap(dev, vma, cpu_addr, dma_addr, size); }
/* * Allocate DMA-coherent memory space and return both the kernel remapped * virtual and bus address for that space. */ void * dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp) { void *memory; if (dma_alloc_from_coherent(dev, size, handle, &memory)) return memory; return __dma_alloc(dev, size, handle, gfp, pgprot_dmacoherent(pgprot_kernel)); }
static int omap_rproc_mmap(struct file *filp, struct vm_area_struct *vma) { vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot); vma->vm_flags |= VM_RESERVED; if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, vma->vm_end - vma->vm_start, vma->vm_page_prot)) { return -EAGAIN; } return 0; }
/* * Initialise the coherent pool for atomic allocations. */ static int __init coherent_init(void) { pgprot_t prot = pgprot_dmacoherent(pgprot_kernel); size_t size = coherent_pool_size; struct page *page; void *ptr; if (!IS_ENABLED(CONFIG_CMA)) return 0; ptr = __alloc_from_contiguous(NULL, size, prot, &page, false); if (ptr) { coherent_head.vm_start = (unsigned long) ptr; coherent_head.vm_end = (unsigned long) ptr + size; printk(KERN_INFO "DMA: preallocated %u KiB pool for atomic coherent allocations\n", (unsigned)size / 1024); return 0; } printk(KERN_ERR "DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", (unsigned)size / 1024); return -ENOMEM; }
struct page *dma_alloc_at_from_contiguous(struct device *dev, int count, unsigned int align, phys_addr_t at_addr) { unsigned long mask, pfn, pageno, start = 0; struct cma *cma = dev_get_cma_area(dev); struct page *page = NULL; int ret; unsigned long start_pfn = __phys_to_pfn(at_addr); if (!cma || !cma->count) return NULL; if (align > CONFIG_CMA_ALIGNMENT) align = CONFIG_CMA_ALIGNMENT; pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma, count, align); if (!count) return NULL; mask = (1 << align) - 1; if (start_pfn && start_pfn < cma->base_pfn) return NULL; start = start_pfn ? start_pfn - cma->base_pfn : start; mutex_lock(&cma_mutex); for (;;) { unsigned long timeout = jiffies + msecs_to_jiffies(8000); pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count, start, count, mask); if (pageno >= cma->count || (start && start != pageno)) break; pfn = cma->base_pfn + pageno; retry: ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA); if (ret == 0) { bitmap_set(cma->bitmap, pageno, count); page = pfn_to_page(pfn); break; } else if (start && time_before(jiffies, timeout)) { cond_resched(); invalidate_bh_lrus(); goto retry; } else if (ret != -EBUSY || start) { break; } pr_debug("%s(): memory range at %p is busy, retrying\n", __func__, pfn_to_page(pfn)); /* try again with a bit different memory target */ start = pageno + mask + 1; } mutex_unlock(&cma_mutex); pr_debug("%s(): returned %p\n", __func__, page); if (page) { __dma_remap(page, count << PAGE_SHIFT, pgprot_dmacoherent(PAGE_KERNEL)); __dma_clear_buffer(page, count << PAGE_SHIFT); } return page; }