static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, struct sg_table *sg, enum dma_data_direction dir) { struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf); dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, dir); sg_free_table(sg); kfree(sg); mutex_lock(&obj->base.dev->struct_mutex); i915_gem_object_unpin_pages(obj); mutex_unlock(&obj->base.dev->struct_mutex); }
static void ion_cma_free(struct ion_buffer *buffer) { struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap); struct device *dev = cma_heap->dev; struct ion_cma_buffer_info *info = buffer->priv_virt; dev_dbg(dev, "Release buffer %p\n", buffer); /* release memory */ dma_free_coherent(dev, buffer->size, info->cpu_addr, info->handle); /* release sg table */ sg_free_table(info->table); kfree(info->table); kfree(info); }
void tee_shm_free(struct tee_shm *shm) { struct tee *tee; if (IS_ERR_OR_NULL(shm)) return; tee = shm->tee; if (tee == NULL) pr_warn("invalid call to tee_shm_free(%p): NULL tee\n", shm); else if (shm->tee == NULL) dev_warn(_DEV(tee), "tee_shm_free(%p): NULL tee\n", shm); else { sg_free_table(&shm->sgt); shm->tee->ops->free(shm); } }
/* This is used for sg_table which is derived from user-pointer */ static void ttm_ub_bo_user_destroy(struct ttm_buffer_object *bo) { struct ttm_bo_user_object *user_bo = container_of(bo, struct ttm_bo_user_object, bo); #if (LINUX_VERSION_CODE > KERNEL_VERSION(3, 3, 0)) if (bo->sg) { ttm_tt_free_user_pages(bo); sg_free_table(bo->sg); kfree(bo->sg); bo->sg = NULL; } #endif ttm_mem_global_free(bo->glob->mem_glob, bo->acc_size); kfree(user_bo); }
static void vb2_dma_sg_put(void *buf_priv) { struct vb2_dma_sg_buf *buf = buf_priv; int i = buf->num_pages; if (atomic_dec_and_test(&buf->refcount)) { dprintk(1, "%s: Freeing buffer of %d pages\n", __func__, buf->num_pages); if (buf->vaddr) vm_unmap_ram(buf->vaddr, buf->num_pages); sg_free_table(&buf->sg_table); while (--i >= 0) __free_page(buf->pages[i]); kfree(buf->pages); kfree(buf); } }
static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk, bool is_rmb) { if (is_rmb) { if (buf_desc->mr_rx[SMC_SINGLE_LINK]) smc_ib_put_memory_region( buf_desc->mr_rx[SMC_SINGLE_LINK]); smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, DMA_FROM_DEVICE); } else { smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, DMA_TO_DEVICE); } sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]); if (buf_desc->cpu_addr) free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order); kfree(buf_desc); }
static void ion_cma_free(struct ion_buffer *buffer) { struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap); struct device *dev = cma_heap->dev; struct ion_cma_buffer_info *info = buffer->priv_virt; dev_dbg(dev, "Release buffer %p\n", buffer); /* release memory */ #ifdef ION_CMA_SPLIT_DMA_ALLOC cma_free_phys((u32)info->handle, buffer->size); #else dma_free_coherent(dev, buffer->size, info->cpu_addr, info->handle); #endif /* release sg table */ sg_free_table(info->table); kfree(info->table); kfree(info); }
static void rockchip_gem_detach_dma_buf(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { struct rockchip_drm_dmabuf_attachment *rockchip_attach = attach->priv; struct sg_table *sgt; if (!rockchip_attach) return; sgt = &rockchip_attach->sgt; if (rockchip_attach->dir != DMA_NONE) dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, rockchip_attach->dir); sg_free_table(sgt); kfree(rockchip_attach); attach->priv = NULL; }
static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf, struct dma_buf_attachment *db_attach) { struct vb2_dma_sg_attachment *attach = db_attach->priv; struct sg_table *sgt; if (!attach) return; sgt = &attach->sgt; /* release the scatterlist cache */ if (attach->dma_dir != DMA_NONE) dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents, attach->dma_dir); sg_free_table(sgt); kfree(attach); db_attach->priv = NULL; }
static void exynos_gem_detach_dma_buf(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { struct exynos_drm_dmabuf_attachment *exynos_attach = attach->priv; struct sg_table *sgt; if (!exynos_attach) return; sgt = &exynos_attach->sgt; if (exynos_attach->dir != DMA_NONE) dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, exynos_attach->dir); sg_free_table(sgt); kfree(exynos_attach); attach->priv = NULL; }
static void ion_mm_heap_free(struct ion_buffer *buffer) { ion_mm_buffer_info* pBufferInfo = (ion_mm_buffer_info*) buffer->priv_virt; ION_FUNC_ENTER; mutex_lock(&ion_mm_buffer_info_mutex); if (pBufferInfo) { if ((pBufferInfo->eModuleID != -1) && (pBufferInfo->MVA)) m4u_dealloc_mva(pBufferInfo->eModuleID, (unsigned int)pBufferInfo->pVA, buffer->size, pBufferInfo->MVA); if (pBufferInfo->pVA) vfree(pBufferInfo->pVA); kfree(pBufferInfo); if (buffer->sg_table) sg_free_table(buffer->sg_table); kfree(buffer->sg_table); } mutex_unlock(&ion_mm_buffer_info_mutex); ION_FUNC_LEAVE; }
static struct sg_table * __i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj, struct page **pvec, int num_pages) { unsigned int max_segment = i915_sg_segment_size(); struct sg_table *st; unsigned int sg_page_sizes; int ret; st = kmalloc(sizeof(*st), GFP_KERNEL); if (!st) return ERR_PTR(-ENOMEM); alloc_table: ret = __sg_alloc_table_from_pages(st, pvec, num_pages, 0, num_pages << PAGE_SHIFT, max_segment, GFP_KERNEL); if (ret) { kfree(st); return ERR_PTR(ret); } ret = i915_gem_gtt_prepare_pages(obj, st); if (ret) { sg_free_table(st); if (max_segment > PAGE_SIZE) { max_segment = PAGE_SIZE; goto alloc_table; } kfree(st); return ERR_PTR(ret); } sg_page_sizes = i915_sg_page_sizes(st->sgl); __i915_gem_object_set_pages(obj, st, sg_page_sizes); return st; }
static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb, struct smc_buf_desc *buf_desc) { struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; if (is_rmb) { if (buf_desc->mr_rx[SMC_SINGLE_LINK]) smc_ib_put_memory_region( buf_desc->mr_rx[SMC_SINGLE_LINK]); smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, DMA_FROM_DEVICE); } else { smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, DMA_TO_DEVICE); } sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]); if (buf_desc->pages) __free_pages(buf_desc->pages, buf_desc->order); kfree(buf_desc); }
/* * @put_userptr: inform the allocator that a USERPTR buffer will no longer * be used */ static void vb2_dma_sg_put_userptr(void *buf_priv) { struct vb2_dma_sg_buf *buf = buf_priv; int i = buf->num_pages; dprintk(1, "%s: Releasing userspace buffer of %d pages\n", __func__, buf->num_pages); if (buf->vaddr) vm_unmap_ram(buf->vaddr, buf->num_pages); sg_free_table(&buf->sg_table); while (--i >= 0) { if (buf->write) set_page_dirty_lock(buf->pages[i]); if (!vma_is_io(buf->vma)) put_page(buf->pages[i]); } kfree(buf->pages); vb2_put_vma(buf->vma); kfree(buf); }
/** * vmw_ttm_unmap_dma - Tear down any TTM page device mappings * * @vmw_tt: Pointer to a struct vmw_ttm_tt * * Tear down any previously set up device DMA mappings and free * any storage space allocated for them. If there are no mappings set up, * this function is a NOP. */ static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt) { struct vmw_private *dev_priv = vmw_tt->dev_priv; if (!vmw_tt->vsgt.sgt) return; switch (dev_priv->map_mode) { case vmw_dma_map_bind: case vmw_dma_map_populate: vmw_ttm_unmap_from_dma(vmw_tt); sg_free_table(vmw_tt->vsgt.sgt); vmw_tt->vsgt.sgt = NULL; ttm_mem_global_free(vmw_mem_glob(dev_priv), vmw_tt->sg_alloc_size); break; default: break; } vmw_tt->mapped = false; }
static void udl_detach_dma_buf(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { struct udl_drm_dmabuf_attachment *udl_attach = attach->priv; struct sg_table *sgt; if (!udl_attach) return; DRM_DEBUG_PRIME("[DEV:%s] size:%zd\n", dev_name(attach->dev), attach->dmabuf->size); sgt = &udl_attach->sgt; if (udl_attach->dir != DMA_NONE) dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, udl_attach->dir); sg_free_table(sgt); kfree(udl_attach); attach->priv = NULL; }
static void exynos_unmap_dmabuf(struct dma_buf_attachment *attach, struct sg_table *sgt) { struct drm_gem_object *obj = attach->dmabuf->priv; struct exynos_drm_gem_obj *exynos_gem_obj; struct exynos_drm_gem_buf *buffer; DRM_DEBUG_KMS("%s\n", __FILE__); exynos_gem_obj = to_exynos_gem_obj(obj); buffer = exynos_gem_obj->buffer; sg_free_table(sgt); kfree(sgt); sgt = NULL; if (atomic_read(&buffer->shared_refcount) <= 0) BUG(); atomic_dec(&buffer->shared_refcount); }
static struct sg_table * tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, enum dma_data_direction dir) { struct drm_gem_object *gem = attach->dmabuf->priv; struct tegra_bo *bo = to_tegra_bo(gem); struct sg_table *sgt; sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) return NULL; if (bo->pages) { struct scatterlist *sg; unsigned int i; if (sg_alloc_table(sgt, bo->num_pages, GFP_KERNEL)) goto free; for_each_sg(sgt->sgl, sg, bo->num_pages, i) sg_set_page(sg, bo->pages[i], PAGE_SIZE, 0); if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) goto free; } else { if (sg_alloc_table(sgt, 1, GFP_KERNEL)) goto free; sg_dma_address(sgt->sgl) = bo->paddr; sg_dma_len(sgt->sgl) = gem->size; } return sgt; free: sg_free_table(sgt); kfree(sgt); return NULL; }
/* * drm_gem_cma_free_object - (struct drm_driver)->gem_free_object callback * function */ void drm_gem_cma_free_object(struct drm_gem_object *gem_obj) { struct drm_gem_cma_object *cma_obj; drm_gem_free_mmap_offset(gem_obj); cma_obj = to_drm_gem_cma_obj(gem_obj); if (cma_obj->vaddr) { dma_free_writecombine(gem_obj->dev->dev, cma_obj->base.size, cma_obj->vaddr, cma_obj->paddr); if (cma_obj->sgt) { sg_free_table(cma_obj->sgt); kfree(cma_obj->sgt); } } else if (gem_obj->import_attach) { drm_prime_gem_destroy(gem_obj, cma_obj->sgt); } drm_gem_object_release(gem_obj); kfree(cma_obj); }
/* * @put_userptr: inform the allocator that a USERPTR buffer will no longer * be used */ static void vb2_dma_sg_put_userptr(void *buf_priv) { struct vb2_dma_sg_buf *buf = buf_priv; struct sg_table *sgt = &buf->sg_table; int i = buf->num_pages; DEFINE_DMA_ATTRS(attrs); dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); dprintk(1, "%s: Releasing userspace buffer of %d pages\n", __func__, buf->num_pages); dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir, &attrs); if (buf->vaddr) vm_unmap_ram(buf->vaddr, buf->num_pages); sg_free_table(buf->dma_sgt); while (--i >= 0) { if (buf->dma_dir == DMA_FROM_DEVICE) set_page_dirty_lock(buf->pages[i]); } vb2_destroy_framevec(buf->vec); kfree(buf); }
dma_addr_t ispmmu_vmap(const struct scatterlist *sglist, int sglen) { int err; void *da; struct sg_table *sgt; unsigned int i; struct scatterlist *sg, *src = (struct scatterlist *)sglist; /* * convert isp sglist to iommu sgt * FIXME: should be fixed in the upper layer? */ sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) return -ENOMEM; err = sg_alloc_table(sgt, sglen, GFP_KERNEL); if (err) goto err_sg_alloc; for_each_sg(sgt->sgl, sg, sgt->nents, i) sg_set_buf(sg, phys_to_virt(sg_dma_address(src + i)), sg_dma_len(src + i)); da = (void *)iommu_vmap(isp_iommu, 0, sgt, IOMMU_FLAG); if (IS_ERR(da)) goto err_vmap; return (dma_addr_t)da; err_vmap: sg_free_table(sgt); err_sg_alloc: kfree(sgt); return -ENOMEM; }
static void put_pages(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); if (msm_obj->pages) { /* For non-cached buffers, ensure the new pages are clean * because display controller, GPU, etc. are not coherent: */ if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED)) dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl, msm_obj->sgt->nents, DMA_BIDIRECTIONAL); sg_free_table(msm_obj->sgt); kfree(msm_obj->sgt); if (use_pages(obj)) drm_gem_put_pages(obj, msm_obj->pages, true, false); else { drm_mm_remove_node(msm_obj->vram_node); drm_free_large(msm_obj->pages); } msm_obj->pages = NULL; } }
static void vb2_dma_sg_put(void *buf_priv) { struct vb2_dma_sg_buf *buf = buf_priv; struct sg_table *sgt = &buf->sg_table; int i = buf->num_pages; if (atomic_dec_and_test(&buf->refcount)) { DEFINE_DMA_ATTRS(attrs); dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); dprintk(1, "%s: Freeing buffer of %d pages\n", __func__, buf->num_pages); dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir, &attrs); if (buf->vaddr) vm_unmap_ram(buf->vaddr, buf->num_pages); sg_free_table(buf->dma_sgt); while (--i >= 0) __free_page(buf->pages[i]); kfree(buf->pages); put_device(buf->dev); kfree(buf); } }
/** * vmw_ttm_map_dma - Make sure TTM pages are visible to the device * * @vmw_tt: Pointer to a struct vmw_ttm_tt * * Select the correct function for and make sure the TTM pages are * visible to the device. Allocate storage for the device mappings. * If a mapping has already been performed, indicated by the storage * pointer being non NULL, the function returns success. */ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) { struct vmw_private *dev_priv = vmw_tt->dev_priv; struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); struct vmw_sg_table *vsgt = &vmw_tt->vsgt; struct vmw_piter iter; dma_addr_t old; int ret = 0; static size_t sgl_size; static size_t sgt_size; if (vmw_tt->mapped) return 0; vsgt->mode = dev_priv->map_mode; vsgt->pages = vmw_tt->dma_ttm.ttm.pages; vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages; vsgt->addrs = vmw_tt->dma_ttm.dma_address; vsgt->sgt = &vmw_tt->sgt; switch (dev_priv->map_mode) { case vmw_dma_map_bind: case vmw_dma_map_populate: if (unlikely(!sgl_size)) { sgl_size = ttm_round_pot(sizeof(struct scatterlist)); sgt_size = ttm_round_pot(sizeof(struct sg_table)); } vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages; ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, false, true); if (unlikely(ret != 0)) return ret; ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, (unsigned long) vsgt->num_pages << PAGE_SHIFT, GFP_KERNEL); if (unlikely(ret != 0)) goto out_sg_alloc_fail; if (vsgt->num_pages > vmw_tt->sgt.nents) { uint64_t over_alloc = sgl_size * (vsgt->num_pages - vmw_tt->sgt.nents); ttm_mem_global_free(glob, over_alloc); vmw_tt->sg_alloc_size -= over_alloc; } ret = vmw_ttm_map_for_dma(vmw_tt); if (unlikely(ret != 0)) goto out_map_fail; break; default: break; } old = ~((dma_addr_t) 0); vmw_tt->vsgt.num_regions = 0; for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) { dma_addr_t cur = vmw_piter_dma_addr(&iter); if (cur != old + PAGE_SIZE) vmw_tt->vsgt.num_regions++; old = cur; } vmw_tt->mapped = true; return 0; out_map_fail: sg_free_table(vmw_tt->vsgt.sgt); vmw_tt->vsgt.sgt = NULL; out_sg_alloc_fail: ttm_mem_global_free(glob, vmw_tt->sg_alloc_size); return ret; }
static struct sg_table * exynos_gem_map_dma_buf(struct dma_buf_attachment *attach, enum dma_data_direction dir) { struct exynos_drm_dmabuf_attachment *exynos_attach = attach->priv; struct exynos_drm_gem_obj *gem_obj = dma_buf_to_obj(attach->dmabuf); struct drm_device *dev = gem_obj->base.dev; struct exynos_drm_gem_buf *buf; struct scatterlist *rd, *wr; struct sg_table *sgt = NULL; unsigned int i; int nents, ret; /* just return current sgt if already requested. */ if (exynos_attach->dir == dir && exynos_attach->is_mapped) return &exynos_attach->sgt; buf = gem_obj->buffer; if (!buf) { DRM_ERROR("buffer is null.\n"); return ERR_PTR(-ENOMEM); } sgt = &exynos_attach->sgt; ret = sg_alloc_table(sgt, buf->sgt->orig_nents, GFP_KERNEL); if (ret) { DRM_ERROR("failed to alloc sgt.\n"); return ERR_PTR(-ENOMEM); } mutex_lock(&dev->struct_mutex); rd = buf->sgt->sgl; wr = sgt->sgl; for (i = 0; i < sgt->orig_nents; ++i) { sg_set_page(wr, sg_page(rd), rd->length, rd->offset); rd = sg_next(rd); wr = sg_next(wr); } if (dir != DMA_NONE) { nents = dma_map_sg(attach->dev, sgt->sgl, sgt->orig_nents, dir); if (!nents) { DRM_ERROR("failed to map sgl with iommu.\n"); sg_free_table(sgt); sgt = ERR_PTR(-EIO); goto err_unlock; } } exynos_attach->is_mapped = true; exynos_attach->dir = dir; attach->priv = exynos_attach; DRM_DEBUG_PRIME("buffer size = 0x%lx\n", buf->size); err_unlock: mutex_unlock(&dev->struct_mutex); return sgt; }
/** * vmw_ttm_map_dma - Make sure TTM pages are visible to the device * * @vmw_tt: Pointer to a struct vmw_ttm_tt * * Select the correct function for and make sure the TTM pages are * visible to the device. Allocate storage for the device mappings. * If a mapping has already been performed, indicated by the storage * pointer being non NULL, the function returns success. */ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) { struct vmw_private *dev_priv = vmw_tt->dev_priv; struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); struct vmw_sg_table *vsgt = &vmw_tt->vsgt; struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false }; struct vmw_piter iter; dma_addr_t old; int ret = 0; static size_t sgl_size; static size_t sgt_size; if (vmw_tt->mapped) return 0; vsgt->mode = dev_priv->map_mode; vsgt->pages = vmw_tt->dma_ttm.ttm.pages; vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages; vsgt->addrs = vmw_tt->dma_ttm.dma_address; vsgt->sgt = &vmw_tt->sgt; switch (dev_priv->map_mode) { case vmw_dma_map_bind: case vmw_dma_map_populate: if (unlikely(!sgl_size)) { sgl_size = ttm_round_pot(sizeof(struct scatterlist)); sgt_size = ttm_round_pot(sizeof(struct sg_table)); } vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages; ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, &ctx); if (unlikely(ret != 0)) return ret; ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, (unsigned long) vsgt->num_pages << PAGE_SHIFT, GFP_KERNEL); if (unlikely(ret != 0)) goto out_sg_alloc_fail; if (vsgt->num_pages > vmw_tt->sgt.nents) { uint64_t over_alloc = sgl_size * (vsgt->num_pages - vmw_tt->sgt.nents); ttm_mem_global_free(glob, over_alloc); vmw_tt->sg_alloc_size -= over_alloc; } ret = vmw_ttm_map_for_dma(vmw_tt); if (unlikely(ret != 0)) goto out_map_fail; break; default: break; } old = ~((dma_addr_t) 0); vmw_tt->vsgt.num_regions = 0; for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) { dma_addr_t cur = vmw_piter_dma_addr(&iter); if (cur != old + PAGE_SIZE) vmw_tt->vsgt.num_regions++; old = cur; } vmw_tt->mapped = true; return 0; out_map_fail: sg_free_table(vmw_tt->vsgt.sgt); vmw_tt->vsgt.sgt = NULL; out_sg_alloc_fail: ttm_mem_global_free(glob, vmw_tt->sg_alloc_size); return ret; } /** * vmw_ttm_unmap_dma - Tear down any TTM page device mappings * * @vmw_tt: Pointer to a struct vmw_ttm_tt * * Tear down any previously set up device DMA mappings and free * any storage space allocated for them. If there are no mappings set up, * this function is a NOP. */ static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt) { struct vmw_private *dev_priv = vmw_tt->dev_priv; if (!vmw_tt->vsgt.sgt) return; switch (dev_priv->map_mode) { case vmw_dma_map_bind: case vmw_dma_map_populate: vmw_ttm_unmap_from_dma(vmw_tt); sg_free_table(vmw_tt->vsgt.sgt); vmw_tt->vsgt.sgt = NULL; ttm_mem_global_free(vmw_mem_glob(dev_priv), vmw_tt->sg_alloc_size); break; default: break; } vmw_tt->mapped = false; } /** * vmw_bo_map_dma - Make sure buffer object pages are visible to the device * * @bo: Pointer to a struct ttm_buffer_object * * Wrapper around vmw_ttm_map_dma, that takes a TTM buffer object pointer * instead of a pointer to a struct vmw_ttm_backend as argument. * Note that the buffer object must be either pinned or reserved before * calling this function. */ int vmw_bo_map_dma(struct ttm_buffer_object *bo) { struct vmw_ttm_tt *vmw_tt = container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm); return vmw_ttm_map_dma(vmw_tt); } /** * vmw_bo_unmap_dma - Make sure buffer object pages are visible to the device * * @bo: Pointer to a struct ttm_buffer_object * * Wrapper around vmw_ttm_unmap_dma, that takes a TTM buffer object pointer * instead of a pointer to a struct vmw_ttm_backend as argument. */ void vmw_bo_unmap_dma(struct ttm_buffer_object *bo) { struct vmw_ttm_tt *vmw_tt = container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm); vmw_ttm_unmap_dma(vmw_tt); } /** * vmw_bo_sg_table - Return a struct vmw_sg_table object for a * TTM buffer object * * @bo: Pointer to a struct ttm_buffer_object * * Returns a pointer to a struct vmw_sg_table object. The object should * not be freed after use. * Note that for the device addresses to be valid, the buffer object must * either be reserved or pinned. */ const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo) { struct vmw_ttm_tt *vmw_tt = container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm); return &vmw_tt->vsgt; } static int vmw_ttm_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) { struct vmw_ttm_tt *vmw_be = container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); int ret; ret = vmw_ttm_map_dma(vmw_be); if (unlikely(ret != 0)) return ret; vmw_be->gmr_id = bo_mem->start; vmw_be->mem_type = bo_mem->mem_type; switch (bo_mem->mem_type) { case VMW_PL_GMR: return vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt, ttm->num_pages, vmw_be->gmr_id); case VMW_PL_MOB: if (unlikely(vmw_be->mob == NULL)) { vmw_be->mob = vmw_mob_create(ttm->num_pages); if (unlikely(vmw_be->mob == NULL)) return -ENOMEM; } return vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob, &vmw_be->vsgt, ttm->num_pages, vmw_be->gmr_id); default: BUG(); } return 0; } static int vmw_ttm_unbind(struct ttm_tt *ttm) { struct vmw_ttm_tt *vmw_be = container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); switch (vmw_be->mem_type) { case VMW_PL_GMR: vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id); break; case VMW_PL_MOB: vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob); break; default: BUG(); } if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind) vmw_ttm_unmap_dma(vmw_be); return 0; } static void vmw_ttm_destroy(struct ttm_tt *ttm) { struct vmw_ttm_tt *vmw_be = container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); vmw_ttm_unmap_dma(vmw_be); if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent) ttm_dma_tt_fini(&vmw_be->dma_ttm); else ttm_tt_fini(ttm); if (vmw_be->mob) vmw_mob_destroy(vmw_be->mob); kfree(vmw_be); } static int vmw_ttm_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); struct vmw_private *dev_priv = vmw_tt->dev_priv; struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); int ret; if (ttm->state != tt_unpopulated) return 0; if (dev_priv->map_mode == vmw_dma_alloc_coherent) { size_t size = ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t)); ret = ttm_mem_global_alloc(glob, size, ctx); if (unlikely(ret != 0)) return ret; ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev, ctx); if (unlikely(ret != 0)) ttm_mem_global_free(glob, size); } else ret = ttm_pool_populate(ttm, ctx); return ret; } static void vmw_ttm_unpopulate(struct ttm_tt *ttm) { struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); struct vmw_private *dev_priv = vmw_tt->dev_priv; struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); if (vmw_tt->mob) { vmw_mob_destroy(vmw_tt->mob); vmw_tt->mob = NULL; } vmw_ttm_unmap_dma(vmw_tt); if (dev_priv->map_mode == vmw_dma_alloc_coherent) { size_t size = ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t)); ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev); ttm_mem_global_free(glob, size); } else ttm_pool_unpopulate(ttm); } static struct ttm_backend_func vmw_ttm_func = { .bind = vmw_ttm_bind, .unbind = vmw_ttm_unbind, .destroy = vmw_ttm_destroy, }; static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) { struct vmw_ttm_tt *vmw_be; int ret; vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL); if (!vmw_be) return NULL; vmw_be->dma_ttm.ttm.func = &vmw_ttm_func; vmw_be->dev_priv = container_of(bo->bdev, struct vmw_private, bdev); vmw_be->mob = NULL; if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent) ret = ttm_dma_tt_init(&vmw_be->dma_ttm, bo, page_flags); else ret = ttm_tt_init(&vmw_be->dma_ttm.ttm, bo, page_flags); if (unlikely(ret != 0)) goto out_no_init; return &vmw_be->dma_ttm.ttm; out_no_init: kfree(vmw_be); return NULL; } static int vmw_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) { return 0; } static int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, struct ttm_mem_type_manager *man) { switch (type) { case TTM_PL_SYSTEM: /* System memory */ man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_CACHED; man->default_caching = TTM_PL_FLAG_CACHED; break; case TTM_PL_VRAM: /* "On-card" video ram */ man->func = &ttm_bo_manager_func; man->gpu_offset = 0; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_CACHED; man->default_caching = TTM_PL_FLAG_CACHED; break; case VMW_PL_GMR: case VMW_PL_MOB: /* * "Guest Memory Regions" is an aperture like feature with * one slot per bo. There is an upper limit of the number of * slots as well as the bo size. */ man->func = &vmw_gmrid_manager_func; man->gpu_offset = 0; man->flags = TTM_MEMTYPE_FLAG_CMA | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_CACHED; man->default_caching = TTM_PL_FLAG_CACHED; break; default: DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); return -EINVAL; } return 0; } static void vmw_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *placement) { *placement = vmw_sys_placement; } static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp) { struct ttm_object_file *tfile = vmw_fpriv((struct drm_file *)filp->private_data)->tfile; return vmw_user_bo_verify_access(bo, tfile); } static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev); mem->bus.addr = NULL; mem->bus.is_iomem = false; mem->bus.offset = 0; mem->bus.size = mem->num_pages << PAGE_SHIFT; mem->bus.base = 0; if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) return -EINVAL; switch (mem->mem_type) { case TTM_PL_SYSTEM: case VMW_PL_GMR: case VMW_PL_MOB: return 0; case TTM_PL_VRAM: mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.base = dev_priv->vram_start; mem->bus.is_iomem = true; break; default: return -EINVAL; } return 0; } static void vmw_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) { } static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) { return 0; } /** * vmw_move_notify - TTM move_notify_callback * * @bo: The TTM buffer object about to move. * @mem: The struct ttm_mem_reg indicating to what memory * region the move is taking place. * * Calls move_notify for all subsystems needing it. * (currently only resources). */ static void vmw_move_notify(struct ttm_buffer_object *bo, bool evict, struct ttm_mem_reg *mem) { vmw_bo_move_notify(bo, mem); vmw_query_move_notify(bo, mem); }
static void *vb2_dma_sg_get_userptr(void *alloc_ctx, unsigned long vaddr, unsigned long size, enum dma_data_direction dma_dir) { struct vb2_dma_sg_conf *conf = alloc_ctx; struct vb2_dma_sg_buf *buf; unsigned long first, last; int num_pages_from_user; struct vm_area_struct *vma; struct sg_table *sgt; DEFINE_DMA_ATTRS(attrs); #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); #endif buf = kzalloc(sizeof *buf, GFP_KERNEL); if (!buf) return NULL; buf->vaddr = NULL; buf->dev = conf->dev; buf->dma_dir = dma_dir; buf->offset = vaddr & ~PAGE_MASK; buf->size = size; buf->dma_sgt = &buf->sg_table; first = (vaddr & PAGE_MASK) >> PAGE_SHIFT; last = ((vaddr + size - 1) & PAGE_MASK) >> PAGE_SHIFT; buf->num_pages = last - first + 1; buf->pages = kzalloc(buf->num_pages * sizeof(struct page *), GFP_KERNEL); if (!buf->pages) goto userptr_fail_alloc_pages; vma = find_vma(current->mm, vaddr); if (!vma) { dprintk(1, "no vma for address %lu\n", vaddr); goto userptr_fail_find_vma; } if (vma->vm_end < vaddr + size) { dprintk(1, "vma at %lu is too small for %lu bytes\n", vaddr, size); goto userptr_fail_find_vma; } buf->vma = vb2_get_vma(vma); if (!buf->vma) { dprintk(1, "failed to copy vma\n"); goto userptr_fail_find_vma; } if (vma_is_io(buf->vma)) { for (num_pages_from_user = 0; num_pages_from_user < buf->num_pages; ++num_pages_from_user, vaddr += PAGE_SIZE) { unsigned long pfn; if (follow_pfn(vma, vaddr, &pfn)) { dprintk(1, "no page for address %lu\n", vaddr); break; } buf->pages[num_pages_from_user] = pfn_to_page(pfn); } } else num_pages_from_user = get_user_pages(current, current->mm, vaddr & PAGE_MASK, buf->num_pages, buf->dma_dir == DMA_FROM_DEVICE, 1, /* force */ buf->pages, NULL); if (num_pages_from_user != buf->num_pages) goto userptr_fail_get_user_pages; if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages, buf->num_pages, buf->offset, size, 0)) goto userptr_fail_alloc_table_from_pages; sgt = &buf->sg_table; /* * No need to sync to the device, this will happen later when the * prepare() memop is called. */ sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir, &attrs); if (!sgt->nents) goto userptr_fail_map; return buf; userptr_fail_map: sg_free_table(&buf->sg_table); userptr_fail_alloc_table_from_pages: userptr_fail_get_user_pages: dprintk(1, "get_user_pages requested/got: %d/%d]\n", buf->num_pages, num_pages_from_user); if (!vma_is_io(buf->vma)) while (--num_pages_from_user >= 0) put_page(buf->pages[num_pages_from_user]); vb2_put_vma(buf->vma); userptr_fail_find_vma: kfree(buf->pages); userptr_fail_alloc_pages: kfree(buf); return NULL; }
static void *vb2_dma_sg_alloc(void *alloc_ctx, unsigned long size, enum dma_data_direction dma_dir, gfp_t gfp_flags) { struct vb2_dma_sg_conf *conf = alloc_ctx; struct vb2_dma_sg_buf *buf; struct sg_table *sgt; int ret; int num_pages; DEFINE_DMA_ATTRS(attrs); #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); #endif if (WARN_ON(alloc_ctx == NULL)) return NULL; buf = kzalloc(sizeof *buf, GFP_KERNEL); if (!buf) return NULL; buf->vaddr = NULL; buf->dma_dir = dma_dir; buf->offset = 0; buf->size = size; /* size is already page aligned */ buf->num_pages = size >> PAGE_SHIFT; buf->dma_sgt = &buf->sg_table; buf->pages = kzalloc(buf->num_pages * sizeof(struct page *), GFP_KERNEL); if (!buf->pages) goto fail_pages_array_alloc; ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags); if (ret) goto fail_pages_alloc; ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages, buf->num_pages, 0, size, GFP_KERNEL); if (ret) goto fail_table_alloc; /* Prevent the device from being released while the buffer is used */ buf->dev = get_device(conf->dev); sgt = &buf->sg_table; /* * No need to sync to the device, this will happen later when the * prepare() memop is called. */ sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir, &attrs); if (!sgt->nents) goto fail_map; buf->handler.refcount = &buf->refcount; buf->handler.put = vb2_dma_sg_put; buf->handler.arg = buf; atomic_inc(&buf->refcount); dprintk(1, "%s: Allocated buffer of %d pages\n", __func__, buf->num_pages); return buf; fail_map: put_device(buf->dev); sg_free_table(buf->dma_sgt); fail_table_alloc: num_pages = buf->num_pages; while (num_pages--) __free_page(buf->pages[num_pages]); fail_pages_alloc: kfree(buf->pages); fail_pages_array_alloc: kfree(buf); return NULL; }
static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj) { /* Should only be called during free */ sg_free_table(obj->pages); kfree(obj->pages); }
static int qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req) { struct ablkcipher_request *req = ablkcipher_request_cast(async_req); struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req); struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm); struct qce_device *qce = tmpl->qce; enum dma_data_direction dir_src, dir_dst; struct scatterlist *sg; bool diff_dst; gfp_t gfp; int ret; rctx->iv = req->info; rctx->ivsize = crypto_ablkcipher_ivsize(ablkcipher); rctx->cryptlen = req->nbytes; diff_dst = (req->src != req->dst) ? true : false; dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); if (diff_dst) rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); else rctx->dst_nents = rctx->src_nents; if (rctx->src_nents < 0) { dev_err(qce->dev, "Invalid numbers of src SG.\n"); return rctx->src_nents; } if (rctx->dst_nents < 0) { dev_err(qce->dev, "Invalid numbers of dst SG.\n"); return -rctx->dst_nents; } rctx->dst_nents += 1; gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); if (ret) return ret; sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); sg = qce_sgtable_add(&rctx->dst_tbl, req->dst); if (IS_ERR(sg)) { ret = PTR_ERR(sg); goto error_free; } sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg); if (IS_ERR(sg)) { ret = PTR_ERR(sg); goto error_free; } sg_mark_end(sg); rctx->dst_sg = rctx->dst_tbl.sgl; ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); if (ret < 0) goto error_free; if (diff_dst) { ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); if (ret < 0) goto error_unmap_dst; rctx->src_sg = req->src; } else { rctx->src_sg = rctx->dst_sg; } ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents, rctx->dst_sg, rctx->dst_nents, qce_ablkcipher_done, async_req); if (ret) goto error_unmap_src; qce_dma_issue_pending(&qce->dma); ret = qce_start(async_req, tmpl->crypto_alg_type, req->nbytes, 0); if (ret) goto error_terminate; return 0; error_terminate: qce_dma_terminate_all(&qce->dma); error_unmap_src: if (diff_dst) dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); error_unmap_dst: dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); error_free: sg_free_table(&rctx->dst_tbl); return ret; }