static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) { struct page *page; void *addr; /* Following is a work-around (a.k.a. hack) to prevent pages * with __GFP_COMP being passed to split_page() which cannot * handle them. The real problem is that this flag probably * should be 0 on ARM as it is not supported on this * platform--see CONFIG_HUGETLB_PAGE. */ gfp &= ~(__GFP_COMP); *handle = ~0; size = PAGE_ALIGN(size); page = __dma_alloc_buffer(dev, size, gfp); if (!page) return NULL; if (!arch_is_coherent()) addr = __dma_alloc_remap(page, size, gfp, prot); else addr = page_address(page); if (addr) *handle = pfn_to_dma(dev, page_to_pfn(page)); else __dma_free_buffer(page, size); return addr; }
/* * see if a buffer address is in an 'unsafe' range. if it is * allocate a 'safe' buffer and copy the unsafe buffer into it. * substitute the safe buffer for the unsafe one. * (basically move the buffer from an unsafe area to a safe one) */ dma_addr_t __dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir) { dma_addr_t dma_addr; int ret; dev_dbg(dev, "%s(page=%p,off=%#lx,size=%zx,dir=%x)\n", __func__, page, offset, size, dir); dma_addr = pfn_to_dma(dev, page_to_pfn(page)) + offset; ret = needs_bounce(dev, dma_addr, size); if (ret < 0) return ~0; if (ret == 0) { __dma_page_cpu_to_dev(page, offset, size, dir); return dma_addr; } if (PageHighMem(page)) { dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n"); return ~0; } return map_single(dev, page_address(page) + offset, size, dir); }
static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) { struct page *page; void *addr; *handle = ~0; size = PAGE_ALIGN(size); page = __dma_alloc_buffer(dev, size, gfp); if (!page) return NULL; if (!arch_is_coherent()) addr = __dma_alloc_remap(page, size, gfp, prot); else addr = page_address(page); if (addr) *handle = pfn_to_dma(dev, page_to_pfn(page)); else __dma_free_buffer(page, size); return addr; }
/** * arm_dma_map_page - map a portion of a page for streaming DMA * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices * @page: page that buffer resides in * @offset: offset into page for start of buffer * @size: size of buffer to map * @dir: DMA transfer direction * * Ensure that any data held in the cache is appropriately discarded * or written back. * * The device owns this memory once this call has completed. The CPU * can regain ownership by calling dma_unmap_page(). */ static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { if (!arch_is_coherent()) __dma_page_cpu_to_dev(page, offset, size, dir); return pfn_to_dma(dev, page_to_pfn(page)) + offset; }
/** * arm_dma_map_page - map a portion of a page for streaming DMA * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices * @page: page that buffer resides in * @offset: offset into page for start of buffer * @size: size of buffer to map * @dir: DMA transfer direction * * Ensure that any data held in the cache is appropriately discarded * or written back. * * The device owns this memory once this call has completed. The CPU * can regain ownership by calling dma_unmap_page(). */ static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) __dma_page_cpu_to_dev(page, offset, size, dir); return pfn_to_dma(dev, page_to_pfn(page)) + offset; }
static dma_addr_t mvebu_hwcc_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { if (dir != DMA_TO_DEVICE) mvebu_hwcc_sync_io_barrier(); return pfn_to_dma(dev, page_to_pfn(page)) + offset; }
static void __isp_stat_buf_sync_magic(struct ispstat *stat, struct ispstat_buffer *buf, u32 buf_size, enum dma_data_direction dir, void (*dma_sync)(struct device *, dma_addr_t, unsigned long, size_t, enum dma_data_direction)) { struct device *dev = stat->isp->dev; struct page *pg; dma_addr_t dma_addr; u32 offset; /* Initial magic words */ pg = vmalloc_to_page(buf->virt_addr); dma_addr = pfn_to_dma(dev, page_to_pfn(pg)); dma_sync(dev, dma_addr, 0, MAGIC_SIZE, dir); /* Final magic words */ pg = vmalloc_to_page(buf->virt_addr + buf_size); dma_addr = pfn_to_dma(dev, page_to_pfn(pg)); offset = ((u32)buf->virt_addr + buf_size) & ~PAGE_MASK; dma_sync(dev, dma_addr, offset, MAGIC_SIZE, dir); }
static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool) { struct page *page = alloc_pages(pool->gfp_mask, pool->order); if (!page) return NULL; /* this is only being used to flush the page for dma, this api is not really suitable for calling from a driver but no better way to flush a page for dma exist at this time */ arm_dma_ops.sync_single_for_device(NULL, pfn_to_dma(NULL, page_to_pfn(page)), PAGE_SIZE << pool->order, DMA_FROM_DEVICE); return page; }
static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool) { struct page *page = alloc_pages(pool->gfp_mask, pool->order); if (!page) return NULL; /* this is only being used to flush the page for dma, this api is not really suitable for calling from a driver but no better way to flush a page for dma exist at this time */ #ifdef CONFIG_64BIT dma_sync_single_for_device(NULL, (dma_addr_t)page_to_phys(page), PAGE_SIZE << pool->order, DMA_BIDIRECTIONAL); #else arm_dma_ops.sync_single_for_device(NULL, pfn_to_dma(NULL, page_to_pfn(page)), PAGE_SIZE << pool->order, DMA_BIDIRECTIONAL); #endif return page; }
/* must be called with write lock held on data->sem */ static int bralloc_mem_allocate(struct file *file, bralloc_data_t *data, unsigned long size) { int alloc_flags; int ret = 0; bralloc_debug("Enter ..\n"); /* Find what type of allocation and mapping we need to do, as they all * need to be consistent with the mmaps we will be creating for * userspace */ if (file->f_flags & O_SYNC) { alloc_flags = BRALLOC_DMA_COHERENT_ALLOC; /* Create uncached allocation, we still need to see if they * need to be write-buffered or not, but we'll add in something * later for that. For now, uncached allocations are coherent * allocations. */ /* Allocate coherent (uncached, strongly ordered) contiguous dma memory here */ data->kaddr = dma_alloc_coherent(&br.pdev->dev, size, &data->dma_handle, GFP_KERNEL); if (!data->kaddr) { bralloc_error("Failed to allocate contiguous DMA memory for (%d/%d), size(%lu)\n", current->pid, current->tgid, size); ret = -ENOMEM; goto done; } data->start_page = virt_to_page(data->kaddr); } else if (file->f_flags & O_DIRECT) { alloc_flags = BRALLOC_DMA_WRITECOMBINE_ALLOC; /* If file is opened with O_DIRECT we consider this writecombine * allocation+mapping (using write buffers). Userspace *must* * lock/unlock region before using it. For us, "lock" means * do nothing, "unlock" means drain write buffer */ /* Allocate coherent (uncached, strongly ordered) contiguous dma memory here */ data->kaddr = dma_alloc_writecombine(&br.pdev->dev, size, &data->dma_handle, GFP_KERNEL); if (!data->kaddr) { bralloc_error("Failed to allocate contiguous DMA memory for (%d/%d), size(%lu)\n", current->pid, current->tgid, size); ret = -ENOMEM; goto done; } data->start_page = virt_to_page(data->kaddr); } else if (file->f_flags & FASYNC) { alloc_flags = BRALLOC_WRITETHROUGH_ALLOC; /* If file is opened with O_ASYNC, we allocate+map using the * Write-Through cached mappings. This means for all files * opened with O_ASYNC, userspace must call lock/unlock ioctls * before using the memory region. For us, "lock" means * "invalidate cachelines" and "unlock" and do nothing */ data->start_page = dma_alloc_from_contiguous(&br.pdev->dev, (size >> PAGE_SHIFT), CONFIG_CMA_ALIGNMENT); if (!data->start_page) { bralloc_error("Failed to allocate contiguous DMA memory for (%d/%d), size(%lu)\n", current->pid, current->tgid, size); ret = -ENOMEM; goto done; } data->kaddr = page_address(data->start_page); /* We must clear the buffer and remap it with WT cachebility to * make sure the userspace mappings and kernel mappings are * identical in terms of memory attributes and cacheability, * otherwise unthinkable things can happen according to ARM TRM */ bralloc_mem_clear_buffer(data->kaddr, size); bralloc_mem_kernel_remap(data->start_page, size, pgprot_writethrough(pgprot_kernel)); data->dma_handle = pfn_to_dma(&br.pdev->dev, page_to_pfn(data->start_page)); } else {
static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { return pfn_to_dma(dev, page_to_pfn(page)) + offset; }