static void *mips_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; struct page *page = NULL; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; gfp = massage_gfp_flags(dev, gfp); if (IS_ENABLED(CONFIG_DMA_CMA) && gfpflags_allow_blocking(gfp)) page = dma_alloc_from_contiguous(dev, count, get_order(size), gfp); if (!page) page = alloc_pages(gfp, get_order(size)); if (!page) return NULL; ret = page_address(page); memset(ret, 0, size); *dma_handle = plat_map_dma_mem(dev, ret, size); if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !plat_device_is_coherent(dev)) { dma_cache_wback_inv((unsigned long) ret, size); ret = UNCAC_ADDR(ret); } return ret; }
void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, struct dma_attrs *attrs) { unsigned long dma_mask; struct page *page = NULL; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; dma_addr_t addr; dma_mask = dma_alloc_coherent_mask(dev, flag); flag |= __GFP_ZERO; again: if (!(flag & GFP_ATOMIC)) page = dma_alloc_from_contiguous(dev, count, get_order(size)); if (!page) page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); if (!page) return NULL; addr = page_to_phys(page); if (addr + size > dma_mask) { __free_pages(page, get_order(size)); if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { flag = (flag & ~GFP_DMA32) | GFP_DMA; goto again; } return NULL; } *dma_addr = addr; return page_address(page); }
static int __init atomic_pool_init(void) { pgprot_t prot = __pgprot(PROT_NORMAL_NC); unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT; struct page *page; void *addr; unsigned int pool_size_order = get_order(atomic_pool_size); if (dev_get_cma_area(NULL)) page = dma_alloc_from_contiguous(NULL, nr_pages, pool_size_order, false); else page = alloc_pages(GFP_DMA32, pool_size_order); if (page) { int ret; void *page_addr = page_address(page); memset(page_addr, 0, atomic_pool_size); __dma_flush_area(page_addr, atomic_pool_size); atomic_pool = gen_pool_create(PAGE_SHIFT, -1); if (!atomic_pool) goto free_page; addr = dma_common_contiguous_remap(page, atomic_pool_size, VM_USERMAP, prot, atomic_pool_init); if (!addr) goto destroy_genpool; ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr, page_to_phys(page), atomic_pool_size, -1); if (ret) goto remove_mapping; gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL); pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", atomic_pool_size / 1024); return 0; } goto out; remove_mapping: dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP); destroy_genpool: gen_pool_destroy(atomic_pool); atomic_pool = NULL; free_page: if (!dma_release_from_contiguous(NULL, page, nr_pages)) __free_pages(page, pool_size_order); out: pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", atomic_pool_size / 1024); return -ENOMEM; }
static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, struct dma_attrs *attrs, bool is_coherent) { unsigned long dma_mask; struct page *page; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; dma_addr_t addr; dma_mask = dma_alloc_coherent_mask(dev, flag); flag |= __GFP_ZERO; again: page = NULL; /* CMA can be used only in the context which permits sleeping */ if (flag & __GFP_WAIT) #ifdef CONFIG_CMA_EXPLICIT_USE if (dma_get_attr(DMA_ATTR_CMA, attrs)) { #endif page = dma_alloc_from_contiguous(dev, count, get_order(size)); #ifdef CONFIG_CMA_EXPLICIT_USE if (!page) return NULL; } #endif /* fallback */ if (!page) page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); if (!page) return NULL; addr = page_to_phys(page); if (addr + size > dma_mask) { __free_pages(page, get_order(size)); if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { flag = (flag & ~GFP_DMA32) | GFP_DMA; goto again; } return NULL; } if (is_coherent == false) __dma_set_pages(page, count, attrs); *dma_addr = addr; return page_address(page); }
void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t flag, unsigned long attrs) { unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page = NULL; /* ignore region speicifiers */ flag &= ~(__GFP_DMA | __GFP_HIGHMEM); if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) flag |= GFP_DMA; if (gfpflags_allow_blocking(flag)) page = dma_alloc_from_contiguous(dev, count, get_order(size), flag & __GFP_NOWARN); if (!page) page = alloc_pages(flag, get_order(size)); if (!page) return NULL; *handle = phys_to_dma(dev, page_to_phys(page)); if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { return page; } #ifdef CONFIG_MMU if (PageHighMem(page)) { void *p; p = dma_common_contiguous_remap(page, size, VM_MAP, pgprot_noncached(PAGE_KERNEL), __builtin_return_address(0)); if (!p) { if (!dma_release_from_contiguous(dev, page, count)) __free_pages(page, get_order(size)); } return p; } #endif BUG_ON(!platform_vaddr_cached(page_address(page))); __invalidate_dcache_range((unsigned long)page_address(page), size); return platform_vaddr_to_uncached(page_address(page)); }
struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; int page_order = get_order(size); struct page *page = NULL; u64 phys_mask; if (attrs & DMA_ATTR_NO_WARN) gfp |= __GFP_NOWARN; /* we always manually zero the memory once we are done: */ gfp &= ~__GFP_ZERO; gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, &phys_mask); again: /* CMA can be used only in the context which permits sleeping */ if (gfpflags_allow_blocking(gfp)) { page = dma_alloc_from_contiguous(dev, count, page_order, gfp & __GFP_NOWARN); if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { dma_release_from_contiguous(dev, page, count); page = NULL; } } if (!page) page = alloc_pages_node(dev_to_node(dev), gfp, page_order); if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { __free_pages(page, page_order); page = NULL; if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_mask < DMA_BIT_MASK(64) && !(gfp & (GFP_DMA32 | GFP_DMA))) { gfp |= GFP_DMA32; goto again; } if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { gfp = (gfp & ~GFP_DMA32) | GFP_DMA; goto again; } } return page; }
u32 cma_alloc_phys(size_t size) { unsigned long order; struct page *page; size_t count; size = PAGE_ALIGN(size); order = get_order(size); count = size >> PAGE_SHIFT; page = dma_alloc_from_contiguous(NULL, count, order); if (!page) return 0; __dma_clear_buffer(page, size); return page_to_phys(page); }
void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, struct dma_attrs *attrs) { unsigned long dma_mask; struct page *page; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; dma_addr_t addr; dma_mask = dma_alloc_coherent_mask(dev, flag); flag &= ~__GFP_ZERO; again: page = NULL; /* CMA can be used only in the context which permits sleeping */ if (flag & __GFP_WAIT) { page = dma_alloc_from_contiguous(dev, count, get_order(size)); if (page && page_to_phys(page) + size > dma_mask) { dma_release_from_contiguous(dev, page, count); page = NULL; } } /* fallback */ if (!page) page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); if (!page) return NULL; addr = page_to_phys(page); if (addr + size > dma_mask) { __free_pages(page, get_order(size)); if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { flag = (flag & ~GFP_DMA32) | GFP_DMA; goto again; } return NULL; } memset(page_address(page), 0, size); *dma_addr = addr; return page_address(page); }
static void __init hisi_cma_dev_init(void) { struct cma *cma; struct page *page = NULL; int i; #ifdef CONFIG_HISI_KERNELDUMP int k; struct page *tmp_page = NULL; #endif for(i = 0 ; i < hisi_cma_area_count; i++){ cma = hisi_cma_areas[i].cma_area; if (cma == NULL) continue; dev_set_cma_area(&hisi_cma_dev[i], cma); hisi_cma_areas[i].dev = &hisi_cma_dev[i]; /* when is 0 mean it is static*/ if (hisi_cma_areas[i].dynamic == 0){ page = dma_alloc_from_contiguous(&hisi_cma_dev[i], cma->count, SZ_1M); #ifdef CONFIG_HISI_KERNELDUMP if (page != NULL) { tmp_page = page; for (k=0;k < cma->count;k++){ SetPageMemDump(tmp_page); tmp_page++; } } #endif if (hisi_cma_areas[i].sec_prot){ create_mapping_late(__pfn_to_phys(cma->base_pfn), __phys_to_virt(__pfn_to_phys(cma->base_pfn)), cma->count * PAGE_SIZE, __pgprot(PROT_DEVICE_nGnRE)); } pr_err("%s:%d page addr 0x%llx size %lu\n", __func__, __LINE__, page_to_phys(page),(cma->count<<PAGE_SHIFT)/SZ_1M); } } }
static void *mips_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs) { void *ret; struct page *page = NULL; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; /* * XXX: seems like the coherent and non-coherent implementations could * be consolidated. */ if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp); gfp = massage_gfp_flags(dev, gfp); if (IS_ENABLED(CONFIG_DMA_CMA) && !(gfp & GFP_ATOMIC)) page = dma_alloc_from_contiguous(dev, count, get_order(size)); if (!page) page = alloc_pages(gfp, get_order(size)); if (!page) return NULL; ret = page_address(page); memset(ret, 0, size); *dma_handle = plat_map_dma_mem(dev, ret, size); if (!plat_device_is_coherent(dev)) { dma_cache_wback_inv((unsigned long) ret, size); if (!hw_coherentio) ret = UNCAC_ADDR(ret); } return ret; }