static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
u64 *phys_mask)
{
if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
dma_mask = dev->bus_dma_mask;
if (force_dma_unencrypted())
*phys_mask = __dma_to_phys(dev, dma_mask);
else
*phys_mask = dma_to_phys(dev, dma_mask);
/*
* Optimistically try the zone that the physical address mask falls
* into first. If that returns memory that isn't actually addressable
* we will fallback to the next lower zone and try again.
*
* Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding
* zones.
*/
if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
return GFP_DMA;
if (*phys_mask <= DMA_BIT_MASK(32))
return GFP_DMA32;
return 0;
}
/*
* Allocates/reserves the Platform memory resources early in the boot process.
* This ignores any resources that are designated IORESOURCE_IO
*/
void __init platform_alloc_bootmem(void)
{
int i;
int total = 0;
/* Get persistent memory data from command line before allocating
* resources. This need to happen before normal command line parsing
* has been done */
pmem_setup_resource();
/* Loop through looking for resources that want a particular address */
for (i = 0; gp_resources[i].flags != 0; i++) {
int size = resource_size(&gp_resources[i]);
if ((gp_resources[i].start != 0) &&
((gp_resources[i].flags & IORESOURCE_MEM) != 0)) {
reserve_bootmem(dma_to_phys(gp_resources[i].start),
size, 0);
total += resource_size(&gp_resources[i]);
pr_info("reserve resource %s at %08x (%u bytes)\n",
gp_resources[i].name, gp_resources[i].start,
resource_size(&gp_resources[i]));
}
}
/* Loop through assigning addresses for those that are left */
for (i = 0; gp_resources[i].flags != 0; i++) {
int size = resource_size(&gp_resources[i]);
if ((gp_resources[i].start == 0) &&
((gp_resources[i].flags & IORESOURCE_MEM) != 0)) {
void *mem = alloc_bootmem_pages(size);
if (mem == NULL)
pr_err("Unable to allocate bootmem pages "
"for %s\n", gp_resources[i].name);
else {
gp_resources[i].start =
phys_to_dma(virt_to_phys(mem));
gp_resources[i].end =
gp_resources[i].start + size - 1;
total += size;
pr_info("allocate resource %s at %08x "
"(%u bytes)\n",
gp_resources[i].name,
gp_resources[i].start, size);
}
}
}
pr_info("Total Platform driver memory allocation: 0x%08x\n", total);
/* indicate resources that are platform I/O related */
for (i = 0; gp_resources[i].flags != 0; i++) {
if ((gp_resources[i].start != 0) &&
((gp_resources[i].flags & IORESOURCE_IO) != 0)) {
pr_info("reserved platform resource %s at %08x\n",
gp_resources[i].name, gp_resources[i].start);
}
}
}
int mei_txe_dma_setup(struct mei_device *dev)
{
struct mei_txe_hw *hw = to_txe_hw(dev);
int err;
err = mei_reserver_dma_acpi(dev);
if (err)
err = mei_alloc_dma(dev);
if (err)
return err;
err = mei_txe_setup_satt2(dev,
dma_to_phys(&dev->pdev->dev, hw->pool_paddr), hw->pool_size);
if (err) {
if (hw->pool_release)
hw->pool_release(hw);
return err;
}
hw->mdev = mei_mm_init(&dev->pdev->dev,
hw->pool_vaddr, hw->pool_paddr, hw->pool_size);
if (IS_ERR_OR_NULL(hw->mdev))
return PTR_ERR(hw->mdev);
return 0;
}
void x86_swiotlb_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_addr,
unsigned long attrs)
{
if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr)))
swiotlb_free_coherent(dev, size, vaddr, dma_addr);
else
dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
}
void __init platform_alloc_bootmem(void)
{
int i;
int total = 0;
pmem_setup_resource();
for (i = 0; gp_resources[i].flags != 0; i++) {
int size = resource_size(&gp_resources[i]);
if ((gp_resources[i].start != 0) &&
((gp_resources[i].flags & IORESOURCE_MEM) != 0)) {
reserve_bootmem(dma_to_phys(gp_resources[i].start),
size, 0);
total += resource_size(&gp_resources[i]);
pr_info("reserve resource %s at %08x (%u bytes)\n",
gp_resources[i].name, gp_resources[i].start,
resource_size(&gp_resources[i]));
}
}
for (i = 0; gp_resources[i].flags != 0; i++) {
int size = resource_size(&gp_resources[i]);
if ((gp_resources[i].start == 0) &&
((gp_resources[i].flags & IORESOURCE_MEM) != 0)) {
void *mem = alloc_bootmem_pages(size);
if (mem == NULL)
pr_err("Unable to allocate bootmem pages "
"for %s\n", gp_resources[i].name);
else {
gp_resources[i].start =
phys_to_dma(virt_to_phys(mem));
gp_resources[i].end =
gp_resources[i].start + size - 1;
total += size;
pr_info("allocate resource %s at %08x "
"(%u bytes)\n",
gp_resources[i].name,
gp_resources[i].start, size);
}
}
}
pr_info("Total Platform driver memory allocation: 0x%08x\n", total);
for (i = 0; gp_resources[i].flags != 0; i++) {
if ((gp_resources[i].start != 0) &&
((gp_resources[i].flags & IORESOURCE_IO) != 0)) {
pr_info("reserved platform resource %s at %08x\n",
gp_resources[i].name, gp_resources[i].start);
}
}
}
void arch_dma_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, unsigned long attrs)
{
if (!__free_from_pool(vaddr, PAGE_ALIGN(size))) {
void *kaddr = phys_to_virt(dma_to_phys(dev, dma_handle));
vunmap(vaddr);
dma_direct_free_pages(dev, size, kaddr, dma_handle, attrs);
}
}
static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
struct memblock_region *reg;
unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
unsigned long max_dma = min;
#ifdef CONFIG_ZONE_DMA
unsigned long max_dma_phys, dma_end;
#endif
memset(zone_size, 0, sizeof(zone_size));
#ifdef CONFIG_ZONE_DMA
#ifdef CONFIG_ZONE_DMA_ALLOW_CUSTOM_SIZE
max_dma_phys = (unsigned long)dma_to_phys(NULL,
(min << PAGE_SHIFT) + ZONE_DMA_SIZE_BYTES + 1);
#else
max_dma_phys = (unsigned long)dma_to_phys(NULL, DMA_BIT_MASK(32) + 1);
#endif /* CONFIG_ZONE_DMA_ALLOW_CUSTOM_SIZE */
max_dma = max(min, min(max, max_dma_phys >> PAGE_SHIFT));
zone_size[ZONE_DMA] = max_dma - min;
#endif /* CONFIG_ZONE_DMA */
zone_size[ZONE_NORMAL] = max - max_dma;
memcpy(zhole_size, zone_size, sizeof(zhole_size));
for_each_memblock(memory, reg) {
unsigned long start = memblock_region_memory_base_pfn(reg);
unsigned long end = memblock_region_memory_end_pfn(reg);
if (start >= max)
continue;
#ifdef CONFIG_ZONE_DMA
if (start < max_dma) {
dma_end = min(end, max_dma);
zhole_size[ZONE_DMA] -= dma_end - start;
}
#endif
if (end > max_dma) {
unsigned long normal_end = min(end, max);
unsigned long normal_start = max(start, max_dma);
zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
}
}
void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
phys_addr_t phys = dma_to_phys(dev, addr);
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
if (unlikely(is_swiotlb_buffer(phys)))
swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
}
void dma_direct_sync_single_for_device(struct device *dev,
dma_addr_t addr, size_t size, enum dma_data_direction dir)
{
phys_addr_t paddr = dma_to_phys(dev, addr);
if (unlikely(is_swiotlb_buffer(paddr)))
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
if (!dev_is_dma_coherent(dev))
arch_sync_dma_for_device(dev, paddr, size, dir);
}
/*
* Create scatter-list for the already allocated DMA buffer.
* This function could be replaced by dma_common_get_sgtable
* as soon as it will avalaible.
*/
static int ion_cma_get_sgtable(struct device *dev, struct sg_table *sgt,
void *cpu_addr, dma_addr_t handle, size_t size)
{
struct page *page = phys_to_page(dma_to_phys(dev, handle));
struct scatterlist *sg;
int ret;
ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
if (unlikely(ret))
return ret;
sg = sgt->sgl;
sg_set_page(sg, page, PAGE_ALIGN(size), 0);
sg_dma_address(sg) = sg_phys(sg);
return 0;
}
static void __dma_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
struct dma_attrs *attrs)
{
bool freed;
phys_addr_t paddr = dma_to_phys(dev, dma_handle);
if (dev == NULL) {
WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
return;
}
freed = dma_release_from_contiguous(dev,
phys_to_page(paddr),
PAGE_ALIGN(size) >> PAGE_SHIFT);
if (!freed)
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
}
static int mei_txe_pci_resume(struct device *device)
{
struct pci_dev *pdev = to_pci_dev(device);
struct mei_device *dev;
struct mei_txe_hw *hw;
int err;
dev = pci_get_drvdata(pdev);
if (!dev)
return -ENODEV;
pci_enable_msi(pdev);
mei_clear_interrupts(dev);
/* request and enable interrupt */
if (pci_dev_msi_enabled(pdev))
err = request_threaded_irq(pdev->irq,
NULL,
mei_txe_irq_thread_handler,
IRQF_ONESHOT, KBUILD_MODNAME, dev);
else
err = request_threaded_irq(pdev->irq,
mei_txe_irq_quick_handler,
mei_txe_irq_thread_handler,
IRQF_SHARED, KBUILD_MODNAME, dev);
if (err) {
dev_err(&pdev->dev, "request_threaded_irq failed: irq = %d.\n",
pdev->irq);
return err;
}
hw = to_txe_hw(dev);
err = mei_txe_setup_satt2(dev,
dma_to_phys(&dev->pdev->dev, hw->pool_paddr), hw->pool_size);
if (err)
return err;
err = mei_restart(dev);
return err;
}
void arch_dma_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, unsigned long attrs)
{
unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
struct page *page;
if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) {
page = vaddr;
} else if (platform_vaddr_uncached(vaddr)) {
page = virt_to_page(platform_vaddr_to_cached(vaddr));
} else {
#ifdef CONFIG_MMU
dma_common_free_remap(vaddr, size, VM_MAP);
#endif
page = pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_handle)));
}
if (!dma_release_from_contiguous(dev, page, count))
__free_pages(page, get_order(size));
}
static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
struct memblock_region *reg;
unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
unsigned long max_dma = min;
memset(zone_size, 0, sizeof(zone_size));
/* 4GB maximum for 32-bit only capable devices */
if (IS_ENABLED(CONFIG_ZONE_DMA)) {
unsigned long max_dma_phys =
(unsigned long)dma_to_phys(NULL, DMA_BIT_MASK(32) + 1);
max_dma = max(min, min(max, max_dma_phys >> PAGE_SHIFT));
zone_size[ZONE_DMA] = max_dma - min;
}
zone_size[ZONE_NORMAL] = max - max_dma;
memcpy(zhole_size, zone_size, sizeof(zhole_size));
for_each_memblock(memory, reg) {
unsigned long start = memblock_region_memory_base_pfn(reg);
unsigned long end = memblock_region_memory_end_pfn(reg);
if (start >= max)
continue;
if (IS_ENABLED(CONFIG_ZONE_DMA) && start < max_dma) {
unsigned long dma_end = min(end, max_dma);
zhole_size[ZONE_DMA] -= dma_end - start;
}
if (end > max_dma) {
unsigned long normal_end = min(end, max);
unsigned long normal_start = max(start, max_dma);
zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
}
}
/**
* mei_probe - Device Initialization Routine
*
* @pdev: PCI device structure
* @ent: entry in mei_txe_pci_tbl
*
* returns 0 on success, <0 on failure.
*/
static int mei_txe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
struct mei_device *dev;
struct mei_txe_hw *hw;
int err;
int i;
/* enable pci dev */
err = pci_enable_device(pdev);
if (err) {
dev_err(&pdev->dev, "failed to enable pci device.\n");
goto end;
}
/* set PCI host mastering */
pci_set_master(pdev);
/* pci request regions for mei driver */
err = pci_request_regions(pdev, KBUILD_MODNAME);
if (err) {
dev_err(&pdev->dev, "failed to get pci regions.\n");
goto disable_device;
}
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(36));
if (err) {
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (err) {
dev_err(&pdev->dev, "No suitable DMA available.\n");
goto release_regions;
}
}
/* allocates and initializes the mei dev structure */
dev = mei_txe_dev_init(pdev);
if (!dev) {
err = -ENOMEM;
goto release_regions;
}
hw = to_txe_hw(dev);
err = mei_reserver_dma_acpi(dev);
if (err)
err = mei_alloc_dma(dev);
if (err)
goto free_device;
/* mapping IO device memory */
for (i = SEC_BAR; i < NUM_OF_MEM_BARS; i++) {
hw->mem_addr[i] = pci_iomap(pdev, i, 0);
if (!hw->mem_addr[i]) {
dev_err(&pdev->dev, "mapping I/O device memory failure.\n");
err = -ENOMEM;
goto free_device;
}
}
pci_enable_msi(pdev);
/* clear spurious interrupts */
mei_clear_interrupts(dev);
/* request and enable interrupt */
if (pci_dev_msi_enabled(pdev))
err = request_threaded_irq(pdev->irq,
NULL,
mei_txe_irq_thread_handler,
IRQF_ONESHOT, KBUILD_MODNAME, dev);
else
err = request_threaded_irq(pdev->irq,
mei_txe_irq_quick_handler,
mei_txe_irq_thread_handler,
IRQF_SHARED, KBUILD_MODNAME, dev);
if (err) {
dev_err(&pdev->dev, "mei: request_threaded_irq failure. irq = %d\n",
pdev->irq);
goto free_device;
}
if (mei_start(dev)) {
dev_err(&pdev->dev, "init hw failure.\n");
err = -ENODEV;
goto release_irq;
}
err = mei_txe_setup_satt2(dev,
dma_to_phys(&dev->pdev->dev, hw->pool_paddr), hw->pool_size);
if (err)
goto release_irq;
err = mei_register(dev);
if (err)
goto release_irq;
pci_set_drvdata(pdev, dev);
hw->mdev = mei_mm_init(&dev->pdev->dev,
hw->pool_vaddr, hw->pool_paddr, hw->pool_size);
if (IS_ERR_OR_NULL(hw->mdev))
goto deregister_mei;
pm_runtime_set_autosuspend_delay(&pdev->dev, MEI_TXI_RPM_TIMEOUT);
pm_runtime_use_autosuspend(&pdev->dev);
pm_runtime_mark_last_busy(&pdev->dev);
/*
* For not wake-able HW runtime pm framework
* can't be used on pci device level.
* Use domain runtime pm callbacks instead.
*/
if (!pci_dev_run_wake(pdev))
mei_txe_set_pm_domain(dev);
pm_runtime_put_noidle(&pdev->dev);
if (!nopg)
pm_runtime_allow(&pdev->dev);
return 0;
deregister_mei:
mei_deregister(dev);
release_irq:
mei_cancel_work(dev);
/* disable interrupts */
mei_disable_interrupts(dev);
free_irq(pdev->irq, dev);
pci_disable_msi(pdev);
free_device:
if (hw->pool_release)
hw->pool_release(hw);
mei_txe_pci_iounmap(pdev, hw);
kfree(dev);
release_regions:
pci_release_regions(pdev);
disable_device:
pci_disable_device(pdev);
end:
dev_err(&pdev->dev, "initialization failed.\n");
return err;
}
long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
dma_addr_t dma_addr)
{
return __phys_to_pfn(dma_to_phys(dev, dma_addr));
}
