static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); return NULL; } if (IS_ENABLED(CONFIG_ZONE_DMA) && dev->coherent_dma_mask <= DMA_BIT_MASK(32)) flags |= GFP_DMA; if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) { struct page *page; void *addr; page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, get_order(size)); if (!page) return NULL; *dma_handle = phys_to_dma(dev, page_to_phys(page)); addr = page_address(page); memset(addr, 0, size); return addr; } else { return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { if (IS_ENABLED(CONFIG_ZONE_DMA) && dev->coherent_dma_mask <= DMA_BIT_MASK(32)) flags |= GFP_DMA; if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) { struct page *page; page = dma_alloc_from_contiguous(dev, PAGE_ALIGN(size) >> PAGE_SHIFT, get_order(size)); if (page) { *dma_handle = phys_to_dma(dev, page_to_phys(page)); return page_address(page); } else if (dev_get_cma_priv_area(dev)) { pr_err("%s: failed to allocate from dma-contiguous\n", __func__); return NULL; } } return swiotlb_alloc_coherent(dev, size, dma_handle, flags); }
void *dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct page *page; void *ret; page = __dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); if (!page) return NULL; if (PageHighMem(page)) { /* * Depending on the cma= arguments and per-arch setup * dma_alloc_from_contiguous could return highmem pages. * Without remapping there is no way to return them here, * so log an error and fail. */ dev_info(dev, "Rejecting highmem page from CMA.\n"); __dma_direct_free_pages(dev, size, page); return NULL; } ret = page_address(page); if (force_dma_unencrypted()) { set_memory_decrypted((unsigned long)ret, 1 << get_order(size)); *dma_handle = __phys_to_dma(dev, page_to_phys(page)); } else { *dma_handle = phys_to_dma(dev, page_to_phys(page)); } memset(ret, 0, size); return ret; }
static inline dma_addr_t phys_to_dma_direct(struct device *dev, phys_addr_t phys) { if (force_dma_unencrypted()) return __phys_to_dma(dev, phys); return phys_to_dma(dev, phys); }
static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); return NULL; } if (IS_ENABLED(CONFIG_ZONE_DMA) && dev->coherent_dma_mask <= DMA_BIT_MASK(32)) flags |= GFP_DMA; if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) { struct page *page; size = PAGE_ALIGN(size); page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, get_order(size)); if (!page) return NULL; *dma_handle = phys_to_dma(dev, page_to_phys(page)); return page_address(page); } else { return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
/* * Allocates/reserves the Platform memory resources early in the boot process. * This ignores any resources that are designated IORESOURCE_IO */ void __init platform_alloc_bootmem(void) { int i; int total = 0; /* Get persistent memory data from command line before allocating * resources. This need to happen before normal command line parsing * has been done */ pmem_setup_resource(); /* Loop through looking for resources that want a particular address */ for (i = 0; gp_resources[i].flags != 0; i++) { int size = resource_size(&gp_resources[i]); if ((gp_resources[i].start != 0) && ((gp_resources[i].flags & IORESOURCE_MEM) != 0)) { reserve_bootmem(dma_to_phys(gp_resources[i].start), size, 0); total += resource_size(&gp_resources[i]); pr_info("reserve resource %s at %08x (%u bytes)\n", gp_resources[i].name, gp_resources[i].start, resource_size(&gp_resources[i])); } } /* Loop through assigning addresses for those that are left */ for (i = 0; gp_resources[i].flags != 0; i++) { int size = resource_size(&gp_resources[i]); if ((gp_resources[i].start == 0) && ((gp_resources[i].flags & IORESOURCE_MEM) != 0)) { void *mem = alloc_bootmem_pages(size); if (mem == NULL) pr_err("Unable to allocate bootmem pages " "for %s\n", gp_resources[i].name); else { gp_resources[i].start = phys_to_dma(virt_to_phys(mem)); gp_resources[i].end = gp_resources[i].start + size - 1; total += size; pr_info("allocate resource %s at %08x " "(%u bytes)\n", gp_resources[i].name, gp_resources[i].start, size); } } } pr_info("Total Platform driver memory allocation: 0x%08x\n", total); /* indicate resources that are platform I/O related */ for (i = 0; gp_resources[i].flags != 0; i++) { if ((gp_resources[i].start != 0) && ((gp_resources[i].flags & IORESOURCE_IO) != 0)) { pr_info("reserved platform resource %s at %08x\n", gp_resources[i].name, gp_resources[i].start); } } }
void __init platform_alloc_bootmem(void) { int i; int total = 0; pmem_setup_resource(); for (i = 0; gp_resources[i].flags != 0; i++) { int size = resource_size(&gp_resources[i]); if ((gp_resources[i].start != 0) && ((gp_resources[i].flags & IORESOURCE_MEM) != 0)) { reserve_bootmem(dma_to_phys(gp_resources[i].start), size, 0); total += resource_size(&gp_resources[i]); pr_info("reserve resource %s at %08x (%u bytes)\n", gp_resources[i].name, gp_resources[i].start, resource_size(&gp_resources[i])); } } for (i = 0; gp_resources[i].flags != 0; i++) { int size = resource_size(&gp_resources[i]); if ((gp_resources[i].start == 0) && ((gp_resources[i].flags & IORESOURCE_MEM) != 0)) { void *mem = alloc_bootmem_pages(size); if (mem == NULL) pr_err("Unable to allocate bootmem pages " "for %s\n", gp_resources[i].name); else { gp_resources[i].start = phys_to_dma(virt_to_phys(mem)); gp_resources[i].end = gp_resources[i].start + size - 1; total += size; pr_info("allocate resource %s at %08x " "(%u bytes)\n", gp_resources[i].name, gp_resources[i].start, size); } } } pr_info("Total Platform driver memory allocation: 0x%08x\n", total); for (i = 0; gp_resources[i].flags != 0; i++) { if ((gp_resources[i].start != 0) && ((gp_resources[i].flags & IORESOURCE_IO) != 0)) { pr_info("reserved platform resource %s at %08x\n", gp_resources[i].name, gp_resources[i].start); } } }
static void __init pmem_setup_resource(void) { struct resource *resource; resource = asic_resource_get("DiagPersistentMemory"); if (resource && pmemaddr && pmemlen) { resource->start = phys_to_dma(pmemaddr - 0x80000000); resource->end = resource->start + pmemlen - 1; pr_info("persistent memory: start=0x%x end=0x%x\n", resource->start, resource->end); } }
static void *__dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { void *ptr, *coherent_ptr; struct page *page; size = PAGE_ALIGN(size); if (!(flags & __GFP_WAIT)) { struct page *page = NULL; void *addr = __alloc_from_pool(size, &page); if (addr) *dma_handle = phys_to_dma(dev, page_to_phys(page)); return addr; } if (IS_ENABLED(CONFIG_DMA_CMA)) { struct page *page; page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, get_order(size)); if (page) { *dma_handle = phys_to_dma(dev, page_to_phys(page)); ptr = page_address(page); } else if (dev_get_cma_priv_area(dev)) { pr_err("%s: failed to allocate from dma-contiguous\n", __func__); return NULL; } else { ptr = __dma_alloc_coherent( dev, size, dma_handle, flags, attrs); } } else {
/* * Set up persistent memory. If we were given values, we patch the array of * resources. Otherwise, persistent memory may be allocated anywhere at all. */ static void __init pmem_setup_resource(void) { struct resource *resource; resource = asic_resource_get("DiagPersistentMemory"); if (resource && pmemaddr && pmemlen) { /* The address provided by bootloader is in kseg0. Convert to * a bus address. */ resource->start = phys_to_dma(pmemaddr - 0x80000000); resource->end = resource->start + pmemlen - 1; pr_info("persistent memory: start=0x%x end=0x%x\n", resource->start, resource->end); } }
void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t flag, unsigned long attrs) { unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page = NULL; /* ignore region speicifiers */ flag &= ~(__GFP_DMA | __GFP_HIGHMEM); if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) flag |= GFP_DMA; if (gfpflags_allow_blocking(flag)) page = dma_alloc_from_contiguous(dev, count, get_order(size), flag & __GFP_NOWARN); if (!page) page = alloc_pages(flag, get_order(size)); if (!page) return NULL; *handle = phys_to_dma(dev, page_to_phys(page)); if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { return page; } #ifdef CONFIG_MMU if (PageHighMem(page)) { void *p; p = dma_common_contiguous_remap(page, size, VM_MAP, pgprot_noncached(PAGE_KERNEL), __builtin_return_address(0)); if (!p) { if (!dma_release_from_contiguous(dev, page, count)) __free_pages(page, get_order(size)); } return p; } #endif BUG_ON(!platform_vaddr_cached(page_address(page))); __invalidate_dcache_range((unsigned long)page_address(page), size); return platform_vaddr_to_uncached(page_address(page)); }
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { phys_addr_t phys = page_to_phys(page) + offset; dma_addr_t dma_addr = phys_to_dma(dev, phys); if (unlikely(!dma_direct_possible(dev, dma_addr, size)) && !swiotlb_map(dev, &phys, &dma_addr, size, dir, attrs)) { report_addr(dev, dma_addr, size); return DMA_MAPPING_ERROR; } if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) arch_sync_dma_for_device(dev, phys, size, dir); return dma_addr; }
void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs) { struct page *page; void *ptr, *coherent_ptr; pgprot_t prot = pgprot_writecombine(PAGE_KERNEL); size = PAGE_ALIGN(size); if (!gfpflags_allow_blocking(flags)) { struct page *page = NULL; void *addr = __alloc_from_pool(size, &page, flags); if (addr) *dma_handle = phys_to_dma(dev, page_to_phys(page)); return addr; } ptr = dma_direct_alloc_pages(dev, size, dma_handle, flags, attrs); if (!ptr) goto no_mem; /* remove any dirty cache lines on the kernel alias */ __dma_flush_area(ptr, size); /* create a coherent mapping */ page = virt_to_page(ptr); coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP, prot, __builtin_return_address(0)); if (!coherent_ptr) goto no_map; return coherent_ptr; no_map: dma_direct_free_pages(dev, size, ptr, *dma_handle, attrs); no_mem: return NULL; }
static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); return NULL; } if (IS_ENABLED(CONFIG_ZONE_DMA) && dev->coherent_dma_mask <= DMA_BIT_MASK(32)) flags |= GFP_DMA; if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) { struct page *page; void *addr; size = PAGE_ALIGN(size); page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, get_order(size)); if (!page) return NULL; *dma_handle = phys_to_dma(dev, page_to_phys(page)); addr = page_address(page); memset(addr, 0, size); if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs) || dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs)) { /* * flush the caches here because we can't later */ __dma_flush_range(addr, addr + size); __dma_remap(page, size, 0, true); } return addr; } else { return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
/* Note that this doesn't work with highmem page */ static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev, volatile void *address) { return phys_to_dma(hwdev, virt_to_phys(address)); }
static dma_addr_t __arm_lpae_dma_addr(struct device *dev, void *pages) { return phys_to_dma(dev, virt_to_phys(pages)); }