static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment,
struct sg_table *sg,
enum dma_data_direction dir)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf);
dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, dir);
sg_free_table(sg);
kfree(sg);
mutex_lock(&obj->base.dev->struct_mutex);
i915_gem_object_unpin_pages(obj);
mutex_unlock(&obj->base.dev->struct_mutex);
}
static void ion_cma_free(struct ion_buffer *buffer)
{
struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap);
struct device *dev = cma_heap->dev;
struct ion_cma_buffer_info *info = buffer->priv_virt;
dev_dbg(dev, "Release buffer %p\n", buffer);
/* release memory */
dma_free_coherent(dev, buffer->size, info->cpu_addr, info->handle);
/* release sg table */
sg_free_table(info->table);
kfree(info->table);
kfree(info);
}
void tee_shm_free(struct tee_shm *shm)
{
struct tee *tee;
if (IS_ERR_OR_NULL(shm))
return;
tee = shm->tee;
if (tee == NULL)
pr_warn("invalid call to tee_shm_free(%p): NULL tee\n", shm);
else if (shm->tee == NULL)
dev_warn(_DEV(tee), "tee_shm_free(%p): NULL tee\n", shm);
else {
sg_free_table(&shm->sgt);
shm->tee->ops->free(shm);
}
}
/* This is used for sg_table which is derived from user-pointer */
static void ttm_ub_bo_user_destroy(struct ttm_buffer_object *bo)
{
struct ttm_bo_user_object *user_bo =
container_of(bo, struct ttm_bo_user_object, bo);
#if (LINUX_VERSION_CODE > KERNEL_VERSION(3, 3, 0))
if (bo->sg) {
ttm_tt_free_user_pages(bo);
sg_free_table(bo->sg);
kfree(bo->sg);
bo->sg = NULL;
}
#endif
ttm_mem_global_free(bo->glob->mem_glob, bo->acc_size);
kfree(user_bo);
}
static void vb2_dma_sg_put(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
int i = buf->num_pages;
if (atomic_dec_and_test(&buf->refcount)) {
dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
buf->num_pages);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(&buf->sg_table);
while (--i >= 0)
__free_page(buf->pages[i]);
kfree(buf->pages);
kfree(buf);
}
}
static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk,
bool is_rmb)
{
if (is_rmb) {
if (buf_desc->mr_rx[SMC_SINGLE_LINK])
smc_ib_put_memory_region(
buf_desc->mr_rx[SMC_SINGLE_LINK]);
smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
DMA_FROM_DEVICE);
} else {
smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
DMA_TO_DEVICE);
}
sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
if (buf_desc->cpu_addr)
free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order);
kfree(buf_desc);
}
static void ion_cma_free(struct ion_buffer *buffer)
{
struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap);
struct device *dev = cma_heap->dev;
struct ion_cma_buffer_info *info = buffer->priv_virt;
dev_dbg(dev, "Release buffer %p\n", buffer);
/* release memory */
#ifdef ION_CMA_SPLIT_DMA_ALLOC
cma_free_phys((u32)info->handle, buffer->size);
#else
dma_free_coherent(dev, buffer->size, info->cpu_addr, info->handle);
#endif
/* release sg table */
sg_free_table(info->table);
kfree(info->table);
kfree(info);
}
static void rockchip_gem_detach_dma_buf(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
struct rockchip_drm_dmabuf_attachment *rockchip_attach = attach->priv;
struct sg_table *sgt;
if (!rockchip_attach)
return;
sgt = &rockchip_attach->sgt;
if (rockchip_attach->dir != DMA_NONE)
dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
rockchip_attach->dir);
sg_free_table(sgt);
kfree(rockchip_attach);
attach->priv = NULL;
}
static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf,
struct dma_buf_attachment *db_attach)
{
struct vb2_dma_sg_attachment *attach = db_attach->priv;
struct sg_table *sgt;
if (!attach)
return;
sgt = &attach->sgt;
/* release the scatterlist cache */
if (attach->dma_dir != DMA_NONE)
dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
attach->dma_dir);
sg_free_table(sgt);
kfree(attach);
db_attach->priv = NULL;
}
static void exynos_gem_detach_dma_buf(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
struct exynos_drm_dmabuf_attachment *exynos_attach = attach->priv;
struct sg_table *sgt;
if (!exynos_attach)
return;
sgt = &exynos_attach->sgt;
if (exynos_attach->dir != DMA_NONE)
dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
exynos_attach->dir);
sg_free_table(sgt);
kfree(exynos_attach);
attach->priv = NULL;
}
static void ion_mm_heap_free(struct ion_buffer *buffer)
{
ion_mm_buffer_info* pBufferInfo = (ion_mm_buffer_info*) buffer->priv_virt;
ION_FUNC_ENTER;
mutex_lock(&ion_mm_buffer_info_mutex);
if (pBufferInfo)
{
if ((pBufferInfo->eModuleID != -1) && (pBufferInfo->MVA))
m4u_dealloc_mva(pBufferInfo->eModuleID, (unsigned int)pBufferInfo->pVA, buffer->size, pBufferInfo->MVA);
if (pBufferInfo->pVA)
vfree(pBufferInfo->pVA);
kfree(pBufferInfo);
if (buffer->sg_table)
sg_free_table(buffer->sg_table);
kfree(buffer->sg_table);
}
mutex_unlock(&ion_mm_buffer_info_mutex);
ION_FUNC_LEAVE;
}
static struct sg_table *
__i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
struct page **pvec, int num_pages)
{
unsigned int max_segment = i915_sg_segment_size();
struct sg_table *st;
unsigned int sg_page_sizes;
int ret;
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (!st)
return ERR_PTR(-ENOMEM);
alloc_table:
ret = __sg_alloc_table_from_pages(st, pvec, num_pages,
0, num_pages << PAGE_SHIFT,
max_segment,
GFP_KERNEL);
if (ret) {
kfree(st);
return ERR_PTR(ret);
}
ret = i915_gem_gtt_prepare_pages(obj, st);
if (ret) {
sg_free_table(st);
if (max_segment > PAGE_SIZE) {
max_segment = PAGE_SIZE;
goto alloc_table;
}
kfree(st);
return ERR_PTR(ret);
}
sg_page_sizes = i915_sg_page_sizes(st->sgl);
__i915_gem_object_set_pages(obj, st, sg_page_sizes);
return st;
}
static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb,
struct smc_buf_desc *buf_desc)
{
struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
if (is_rmb) {
if (buf_desc->mr_rx[SMC_SINGLE_LINK])
smc_ib_put_memory_region(
buf_desc->mr_rx[SMC_SINGLE_LINK]);
smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
DMA_FROM_DEVICE);
} else {
smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
DMA_TO_DEVICE);
}
sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
if (buf_desc->pages)
__free_pages(buf_desc->pages, buf_desc->order);
kfree(buf_desc);
}
/*
* @put_userptr: inform the allocator that a USERPTR buffer will no longer
* be used
*/
static void vb2_dma_sg_put_userptr(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
int i = buf->num_pages;
dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
__func__, buf->num_pages);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(&buf->sg_table);
while (--i >= 0) {
if (buf->write)
set_page_dirty_lock(buf->pages[i]);
if (!vma_is_io(buf->vma))
put_page(buf->pages[i]);
}
kfree(buf->pages);
vb2_put_vma(buf->vma);
kfree(buf);
}
/**
* vmw_ttm_unmap_dma - Tear down any TTM page device mappings
*
* @vmw_tt: Pointer to a struct vmw_ttm_tt
*
* Tear down any previously set up device DMA mappings and free
* any storage space allocated for them. If there are no mappings set up,
* this function is a NOP.
*/
static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
{
struct vmw_private *dev_priv = vmw_tt->dev_priv;
if (!vmw_tt->vsgt.sgt)
return;
switch (dev_priv->map_mode) {
case vmw_dma_map_bind:
case vmw_dma_map_populate:
vmw_ttm_unmap_from_dma(vmw_tt);
sg_free_table(vmw_tt->vsgt.sgt);
vmw_tt->vsgt.sgt = NULL;
ttm_mem_global_free(vmw_mem_glob(dev_priv),
vmw_tt->sg_alloc_size);
break;
default:
break;
}
vmw_tt->mapped = false;
}
static void udl_detach_dma_buf(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
struct udl_drm_dmabuf_attachment *udl_attach = attach->priv;
struct sg_table *sgt;
if (!udl_attach)
return;
DRM_DEBUG_PRIME("[DEV:%s] size:%zd\n", dev_name(attach->dev),
attach->dmabuf->size);
sgt = &udl_attach->sgt;
if (udl_attach->dir != DMA_NONE)
dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
udl_attach->dir);
sg_free_table(sgt);
kfree(udl_attach);
attach->priv = NULL;
}
static void exynos_unmap_dmabuf(struct dma_buf_attachment *attach,
struct sg_table *sgt)
{
struct drm_gem_object *obj = attach->dmabuf->priv;
struct exynos_drm_gem_obj *exynos_gem_obj;
struct exynos_drm_gem_buf *buffer;
DRM_DEBUG_KMS("%s\n", __FILE__);
exynos_gem_obj = to_exynos_gem_obj(obj);
buffer = exynos_gem_obj->buffer;
sg_free_table(sgt);
kfree(sgt);
sgt = NULL;
if (atomic_read(&buffer->shared_refcount) <= 0)
BUG();
atomic_dec(&buffer->shared_refcount);
}
static struct sg_table *
tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
enum dma_data_direction dir)
{
struct drm_gem_object *gem = attach->dmabuf->priv;
struct tegra_bo *bo = to_tegra_bo(gem);
struct sg_table *sgt;
sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
if (!sgt)
return NULL;
if (bo->pages) {
struct scatterlist *sg;
unsigned int i;
if (sg_alloc_table(sgt, bo->num_pages, GFP_KERNEL))
goto free;
for_each_sg(sgt->sgl, sg, bo->num_pages, i)
sg_set_page(sg, bo->pages[i], PAGE_SIZE, 0);
if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0)
goto free;
} else {
if (sg_alloc_table(sgt, 1, GFP_KERNEL))
goto free;
sg_dma_address(sgt->sgl) = bo->paddr;
sg_dma_len(sgt->sgl) = gem->size;
}
return sgt;
free:
sg_free_table(sgt);
kfree(sgt);
return NULL;
}
/*
* drm_gem_cma_free_object - (struct drm_driver)->gem_free_object callback
* function
*/
void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
{
struct drm_gem_cma_object *cma_obj;
drm_gem_free_mmap_offset(gem_obj);
cma_obj = to_drm_gem_cma_obj(gem_obj);
if (cma_obj->vaddr) {
dma_free_writecombine(gem_obj->dev->dev, cma_obj->base.size,
cma_obj->vaddr, cma_obj->paddr);
if (cma_obj->sgt) {
sg_free_table(cma_obj->sgt);
kfree(cma_obj->sgt);
}
} else if (gem_obj->import_attach) {
drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
}
drm_gem_object_release(gem_obj);
kfree(cma_obj);
}
/*
* @put_userptr: inform the allocator that a USERPTR buffer will no longer
* be used
*/
static void vb2_dma_sg_put_userptr(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
struct sg_table *sgt = &buf->sg_table;
int i = buf->num_pages;
DEFINE_DMA_ATTRS(attrs);
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
__func__, buf->num_pages);
dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir,
&attrs);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(buf->dma_sgt);
while (--i >= 0) {
if (buf->dma_dir == DMA_FROM_DEVICE)
set_page_dirty_lock(buf->pages[i]);
}
vb2_destroy_framevec(buf->vec);
kfree(buf);
}
dma_addr_t ispmmu_vmap(const struct scatterlist *sglist,
int sglen)
{
int err;
void *da;
struct sg_table *sgt;
unsigned int i;
struct scatterlist *sg, *src = (struct scatterlist *)sglist;
/*
* convert isp sglist to iommu sgt
* FIXME: should be fixed in the upper layer?
*/
sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
if (!sgt)
return -ENOMEM;
err = sg_alloc_table(sgt, sglen, GFP_KERNEL);
if (err)
goto err_sg_alloc;
for_each_sg(sgt->sgl, sg, sgt->nents, i)
sg_set_buf(sg, phys_to_virt(sg_dma_address(src + i)),
sg_dma_len(src + i));
da = (void *)iommu_vmap(isp_iommu, 0, sgt, IOMMU_FLAG);
if (IS_ERR(da))
goto err_vmap;
return (dma_addr_t)da;
err_vmap:
sg_free_table(sgt);
err_sg_alloc:
kfree(sgt);
return -ENOMEM;
}
static void put_pages(struct drm_gem_object *obj)
{
struct msm_gem_object *msm_obj = to_msm_bo(obj);
if (msm_obj->pages) {
/* For non-cached buffers, ensure the new pages are clean
* because display controller, GPU, etc. are not coherent:
*/
if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
sg_free_table(msm_obj->sgt);
kfree(msm_obj->sgt);
if (use_pages(obj))
drm_gem_put_pages(obj, msm_obj->pages, true, false);
else {
drm_mm_remove_node(msm_obj->vram_node);
drm_free_large(msm_obj->pages);
}
msm_obj->pages = NULL;
}
}
static void vb2_dma_sg_put(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
struct sg_table *sgt = &buf->sg_table;
int i = buf->num_pages;
if (atomic_dec_and_test(&buf->refcount)) {
DEFINE_DMA_ATTRS(attrs);
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
buf->num_pages);
dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
buf->dma_dir, &attrs);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(buf->dma_sgt);
while (--i >= 0)
__free_page(buf->pages[i]);
kfree(buf->pages);
put_device(buf->dev);
kfree(buf);
}
}
/**
* vmw_ttm_map_dma - Make sure TTM pages are visible to the device
*
* @vmw_tt: Pointer to a struct vmw_ttm_tt
*
* Select the correct function for and make sure the TTM pages are
* visible to the device. Allocate storage for the device mappings.
* If a mapping has already been performed, indicated by the storage
* pointer being non NULL, the function returns success.
*/
static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
{
struct vmw_private *dev_priv = vmw_tt->dev_priv;
struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
struct vmw_piter iter;
dma_addr_t old;
int ret = 0;
static size_t sgl_size;
static size_t sgt_size;
if (vmw_tt->mapped)
return 0;
vsgt->mode = dev_priv->map_mode;
vsgt->pages = vmw_tt->dma_ttm.ttm.pages;
vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages;
vsgt->addrs = vmw_tt->dma_ttm.dma_address;
vsgt->sgt = &vmw_tt->sgt;
switch (dev_priv->map_mode) {
case vmw_dma_map_bind:
case vmw_dma_map_populate:
if (unlikely(!sgl_size)) {
sgl_size = ttm_round_pot(sizeof(struct scatterlist));
sgt_size = ttm_round_pot(sizeof(struct sg_table));
}
vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages;
ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, false,
true);
if (unlikely(ret != 0))
return ret;
ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages,
vsgt->num_pages, 0,
(unsigned long)
vsgt->num_pages << PAGE_SHIFT,
GFP_KERNEL);
if (unlikely(ret != 0))
goto out_sg_alloc_fail;
if (vsgt->num_pages > vmw_tt->sgt.nents) {
uint64_t over_alloc =
sgl_size * (vsgt->num_pages -
vmw_tt->sgt.nents);
ttm_mem_global_free(glob, over_alloc);
vmw_tt->sg_alloc_size -= over_alloc;
}
ret = vmw_ttm_map_for_dma(vmw_tt);
if (unlikely(ret != 0))
goto out_map_fail;
break;
default:
break;
}
old = ~((dma_addr_t) 0);
vmw_tt->vsgt.num_regions = 0;
for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) {
dma_addr_t cur = vmw_piter_dma_addr(&iter);
if (cur != old + PAGE_SIZE)
vmw_tt->vsgt.num_regions++;
old = cur;
}
vmw_tt->mapped = true;
return 0;
out_map_fail:
sg_free_table(vmw_tt->vsgt.sgt);
vmw_tt->vsgt.sgt = NULL;
out_sg_alloc_fail:
ttm_mem_global_free(glob, vmw_tt->sg_alloc_size);
return ret;
}
static struct sg_table *
exynos_gem_map_dma_buf(struct dma_buf_attachment *attach,
enum dma_data_direction dir)
{
struct exynos_drm_dmabuf_attachment *exynos_attach = attach->priv;
struct exynos_drm_gem_obj *gem_obj = dma_buf_to_obj(attach->dmabuf);
struct drm_device *dev = gem_obj->base.dev;
struct exynos_drm_gem_buf *buf;
struct scatterlist *rd, *wr;
struct sg_table *sgt = NULL;
unsigned int i;
int nents, ret;
/* just return current sgt if already requested. */
if (exynos_attach->dir == dir && exynos_attach->is_mapped)
return &exynos_attach->sgt;
buf = gem_obj->buffer;
if (!buf) {
DRM_ERROR("buffer is null.\n");
return ERR_PTR(-ENOMEM);
}
sgt = &exynos_attach->sgt;
ret = sg_alloc_table(sgt, buf->sgt->orig_nents, GFP_KERNEL);
if (ret) {
DRM_ERROR("failed to alloc sgt.\n");
return ERR_PTR(-ENOMEM);
}
mutex_lock(&dev->struct_mutex);
rd = buf->sgt->sgl;
wr = sgt->sgl;
for (i = 0; i < sgt->orig_nents; ++i) {
sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
rd = sg_next(rd);
wr = sg_next(wr);
}
if (dir != DMA_NONE) {
nents = dma_map_sg(attach->dev, sgt->sgl, sgt->orig_nents, dir);
if (!nents) {
DRM_ERROR("failed to map sgl with iommu.\n");
sg_free_table(sgt);
sgt = ERR_PTR(-EIO);
goto err_unlock;
}
}
exynos_attach->is_mapped = true;
exynos_attach->dir = dir;
attach->priv = exynos_attach;
DRM_DEBUG_PRIME("buffer size = 0x%lx\n", buf->size);
err_unlock:
mutex_unlock(&dev->struct_mutex);
return sgt;
}
/**
* vmw_ttm_map_dma - Make sure TTM pages are visible to the device
*
* @vmw_tt: Pointer to a struct vmw_ttm_tt
*
* Select the correct function for and make sure the TTM pages are
* visible to the device. Allocate storage for the device mappings.
* If a mapping has already been performed, indicated by the storage
* pointer being non NULL, the function returns success.
*/
static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
{
struct vmw_private *dev_priv = vmw_tt->dev_priv;
struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
struct ttm_operation_ctx ctx = {
.interruptible = true,
.no_wait_gpu = false
};
struct vmw_piter iter;
dma_addr_t old;
int ret = 0;
static size_t sgl_size;
static size_t sgt_size;
if (vmw_tt->mapped)
return 0;
vsgt->mode = dev_priv->map_mode;
vsgt->pages = vmw_tt->dma_ttm.ttm.pages;
vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages;
vsgt->addrs = vmw_tt->dma_ttm.dma_address;
vsgt->sgt = &vmw_tt->sgt;
switch (dev_priv->map_mode) {
case vmw_dma_map_bind:
case vmw_dma_map_populate:
if (unlikely(!sgl_size)) {
sgl_size = ttm_round_pot(sizeof(struct scatterlist));
sgt_size = ttm_round_pot(sizeof(struct sg_table));
}
vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages;
ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, &ctx);
if (unlikely(ret != 0))
return ret;
ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages,
vsgt->num_pages, 0,
(unsigned long)
vsgt->num_pages << PAGE_SHIFT,
GFP_KERNEL);
if (unlikely(ret != 0))
goto out_sg_alloc_fail;
if (vsgt->num_pages > vmw_tt->sgt.nents) {
uint64_t over_alloc =
sgl_size * (vsgt->num_pages -
vmw_tt->sgt.nents);
ttm_mem_global_free(glob, over_alloc);
vmw_tt->sg_alloc_size -= over_alloc;
}
ret = vmw_ttm_map_for_dma(vmw_tt);
if (unlikely(ret != 0))
goto out_map_fail;
break;
default:
break;
}
old = ~((dma_addr_t) 0);
vmw_tt->vsgt.num_regions = 0;
for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) {
dma_addr_t cur = vmw_piter_dma_addr(&iter);
if (cur != old + PAGE_SIZE)
vmw_tt->vsgt.num_regions++;
old = cur;
}
vmw_tt->mapped = true;
return 0;
out_map_fail:
sg_free_table(vmw_tt->vsgt.sgt);
vmw_tt->vsgt.sgt = NULL;
out_sg_alloc_fail:
ttm_mem_global_free(glob, vmw_tt->sg_alloc_size);
return ret;
}
/**
* vmw_ttm_unmap_dma - Tear down any TTM page device mappings
*
* @vmw_tt: Pointer to a struct vmw_ttm_tt
*
* Tear down any previously set up device DMA mappings and free
* any storage space allocated for them. If there are no mappings set up,
* this function is a NOP.
*/
static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
{
struct vmw_private *dev_priv = vmw_tt->dev_priv;
if (!vmw_tt->vsgt.sgt)
return;
switch (dev_priv->map_mode) {
case vmw_dma_map_bind:
case vmw_dma_map_populate:
vmw_ttm_unmap_from_dma(vmw_tt);
sg_free_table(vmw_tt->vsgt.sgt);
vmw_tt->vsgt.sgt = NULL;
ttm_mem_global_free(vmw_mem_glob(dev_priv),
vmw_tt->sg_alloc_size);
break;
default:
break;
}
vmw_tt->mapped = false;
}
/**
* vmw_bo_map_dma - Make sure buffer object pages are visible to the device
*
* @bo: Pointer to a struct ttm_buffer_object
*
* Wrapper around vmw_ttm_map_dma, that takes a TTM buffer object pointer
* instead of a pointer to a struct vmw_ttm_backend as argument.
* Note that the buffer object must be either pinned or reserved before
* calling this function.
*/
int vmw_bo_map_dma(struct ttm_buffer_object *bo)
{
struct vmw_ttm_tt *vmw_tt =
container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
return vmw_ttm_map_dma(vmw_tt);
}
/**
* vmw_bo_unmap_dma - Make sure buffer object pages are visible to the device
*
* @bo: Pointer to a struct ttm_buffer_object
*
* Wrapper around vmw_ttm_unmap_dma, that takes a TTM buffer object pointer
* instead of a pointer to a struct vmw_ttm_backend as argument.
*/
void vmw_bo_unmap_dma(struct ttm_buffer_object *bo)
{
struct vmw_ttm_tt *vmw_tt =
container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
vmw_ttm_unmap_dma(vmw_tt);
}
/**
* vmw_bo_sg_table - Return a struct vmw_sg_table object for a
* TTM buffer object
*
* @bo: Pointer to a struct ttm_buffer_object
*
* Returns a pointer to a struct vmw_sg_table object. The object should
* not be freed after use.
* Note that for the device addresses to be valid, the buffer object must
* either be reserved or pinned.
*/
const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo)
{
struct vmw_ttm_tt *vmw_tt =
container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
return &vmw_tt->vsgt;
}
static int vmw_ttm_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
{
struct vmw_ttm_tt *vmw_be =
container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
int ret;
ret = vmw_ttm_map_dma(vmw_be);
if (unlikely(ret != 0))
return ret;
vmw_be->gmr_id = bo_mem->start;
vmw_be->mem_type = bo_mem->mem_type;
switch (bo_mem->mem_type) {
case VMW_PL_GMR:
return vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt,
ttm->num_pages, vmw_be->gmr_id);
case VMW_PL_MOB:
if (unlikely(vmw_be->mob == NULL)) {
vmw_be->mob =
vmw_mob_create(ttm->num_pages);
if (unlikely(vmw_be->mob == NULL))
return -ENOMEM;
}
return vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob,
&vmw_be->vsgt, ttm->num_pages,
vmw_be->gmr_id);
default:
BUG();
}
return 0;
}
static int vmw_ttm_unbind(struct ttm_tt *ttm)
{
struct vmw_ttm_tt *vmw_be =
container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
switch (vmw_be->mem_type) {
case VMW_PL_GMR:
vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
break;
case VMW_PL_MOB:
vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob);
break;
default:
BUG();
}
if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind)
vmw_ttm_unmap_dma(vmw_be);
return 0;
}
static void vmw_ttm_destroy(struct ttm_tt *ttm)
{
struct vmw_ttm_tt *vmw_be =
container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
vmw_ttm_unmap_dma(vmw_be);
if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
ttm_dma_tt_fini(&vmw_be->dma_ttm);
else
ttm_tt_fini(ttm);
if (vmw_be->mob)
vmw_mob_destroy(vmw_be->mob);
kfree(vmw_be);
}
static int vmw_ttm_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
{
struct vmw_ttm_tt *vmw_tt =
container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
struct vmw_private *dev_priv = vmw_tt->dev_priv;
struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
int ret;
if (ttm->state != tt_unpopulated)
return 0;
if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
size_t size =
ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
ret = ttm_mem_global_alloc(glob, size, ctx);
if (unlikely(ret != 0))
return ret;
ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev,
ctx);
if (unlikely(ret != 0))
ttm_mem_global_free(glob, size);
} else
ret = ttm_pool_populate(ttm, ctx);
return ret;
}
static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
{
struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
dma_ttm.ttm);
struct vmw_private *dev_priv = vmw_tt->dev_priv;
struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
if (vmw_tt->mob) {
vmw_mob_destroy(vmw_tt->mob);
vmw_tt->mob = NULL;
}
vmw_ttm_unmap_dma(vmw_tt);
if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
size_t size =
ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
ttm_mem_global_free(glob, size);
} else
ttm_pool_unpopulate(ttm);
}
static struct ttm_backend_func vmw_ttm_func = {
.bind = vmw_ttm_bind,
.unbind = vmw_ttm_unbind,
.destroy = vmw_ttm_destroy,
};
static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo,
uint32_t page_flags)
{
struct vmw_ttm_tt *vmw_be;
int ret;
vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
if (!vmw_be)
return NULL;
vmw_be->dma_ttm.ttm.func = &vmw_ttm_func;
vmw_be->dev_priv = container_of(bo->bdev, struct vmw_private, bdev);
vmw_be->mob = NULL;
if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
ret = ttm_dma_tt_init(&vmw_be->dma_ttm, bo, page_flags);
else
ret = ttm_tt_init(&vmw_be->dma_ttm.ttm, bo, page_flags);
if (unlikely(ret != 0))
goto out_no_init;
return &vmw_be->dma_ttm.ttm;
out_no_init:
kfree(vmw_be);
return NULL;
}
static int vmw_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
{
return 0;
}
static int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
struct ttm_mem_type_manager *man)
{
switch (type) {
case TTM_PL_SYSTEM:
/* System memory */
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_CACHED;
man->default_caching = TTM_PL_FLAG_CACHED;
break;
case TTM_PL_VRAM:
/* "On-card" video ram */
man->func = &ttm_bo_manager_func;
man->gpu_offset = 0;
man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_CACHED;
man->default_caching = TTM_PL_FLAG_CACHED;
break;
case VMW_PL_GMR:
case VMW_PL_MOB:
/*
* "Guest Memory Regions" is an aperture like feature with
* one slot per bo. There is an upper limit of the number of
* slots as well as the bo size.
*/
man->func = &vmw_gmrid_manager_func;
man->gpu_offset = 0;
man->flags = TTM_MEMTYPE_FLAG_CMA | TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_CACHED;
man->default_caching = TTM_PL_FLAG_CACHED;
break;
default:
DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
return -EINVAL;
}
return 0;
}
static void vmw_evict_flags(struct ttm_buffer_object *bo,
struct ttm_placement *placement)
{
*placement = vmw_sys_placement;
}
static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
struct ttm_object_file *tfile =
vmw_fpriv((struct drm_file *)filp->private_data)->tfile;
return vmw_user_bo_verify_access(bo, tfile);
}
static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);
mem->bus.addr = NULL;
mem->bus.is_iomem = false;
mem->bus.offset = 0;
mem->bus.size = mem->num_pages << PAGE_SHIFT;
mem->bus.base = 0;
if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
return -EINVAL;
switch (mem->mem_type) {
case TTM_PL_SYSTEM:
case VMW_PL_GMR:
case VMW_PL_MOB:
return 0;
case TTM_PL_VRAM:
mem->bus.offset = mem->start << PAGE_SHIFT;
mem->bus.base = dev_priv->vram_start;
mem->bus.is_iomem = true;
break;
default:
return -EINVAL;
}
return 0;
}
static void vmw_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
}
static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
{
return 0;
}
/**
* vmw_move_notify - TTM move_notify_callback
*
* @bo: The TTM buffer object about to move.
* @mem: The struct ttm_mem_reg indicating to what memory
* region the move is taking place.
*
* Calls move_notify for all subsystems needing it.
* (currently only resources).
*/
static void vmw_move_notify(struct ttm_buffer_object *bo,
bool evict,
struct ttm_mem_reg *mem)
{
vmw_bo_move_notify(bo, mem);
vmw_query_move_notify(bo, mem);
}
static void *vb2_dma_sg_get_userptr(void *alloc_ctx, unsigned long vaddr,
unsigned long size,
enum dma_data_direction dma_dir)
{
struct vb2_dma_sg_conf *conf = alloc_ctx;
struct vb2_dma_sg_buf *buf;
unsigned long first, last;
int num_pages_from_user;
struct vm_area_struct *vma;
struct sg_table *sgt;
DEFINE_DMA_ATTRS(attrs);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0)
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
#endif
buf = kzalloc(sizeof *buf, GFP_KERNEL);
if (!buf)
return NULL;
buf->vaddr = NULL;
buf->dev = conf->dev;
buf->dma_dir = dma_dir;
buf->offset = vaddr & ~PAGE_MASK;
buf->size = size;
buf->dma_sgt = &buf->sg_table;
first = (vaddr & PAGE_MASK) >> PAGE_SHIFT;
last = ((vaddr + size - 1) & PAGE_MASK) >> PAGE_SHIFT;
buf->num_pages = last - first + 1;
buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),
GFP_KERNEL);
if (!buf->pages)
goto userptr_fail_alloc_pages;
vma = find_vma(current->mm, vaddr);
if (!vma) {
dprintk(1, "no vma for address %lu\n", vaddr);
goto userptr_fail_find_vma;
}
if (vma->vm_end < vaddr + size) {
dprintk(1, "vma at %lu is too small for %lu bytes\n",
vaddr, size);
goto userptr_fail_find_vma;
}
buf->vma = vb2_get_vma(vma);
if (!buf->vma) {
dprintk(1, "failed to copy vma\n");
goto userptr_fail_find_vma;
}
if (vma_is_io(buf->vma)) {
for (num_pages_from_user = 0;
num_pages_from_user < buf->num_pages;
++num_pages_from_user, vaddr += PAGE_SIZE) {
unsigned long pfn;
if (follow_pfn(vma, vaddr, &pfn)) {
dprintk(1, "no page for address %lu\n", vaddr);
break;
}
buf->pages[num_pages_from_user] = pfn_to_page(pfn);
}
} else
num_pages_from_user = get_user_pages(current, current->mm,
vaddr & PAGE_MASK,
buf->num_pages,
buf->dma_dir == DMA_FROM_DEVICE,
1, /* force */
buf->pages,
NULL);
if (num_pages_from_user != buf->num_pages)
goto userptr_fail_get_user_pages;
if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
buf->num_pages, buf->offset, size, 0))
goto userptr_fail_alloc_table_from_pages;
sgt = &buf->sg_table;
/*
* No need to sync to the device, this will happen later when the
* prepare() memop is called.
*/
sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
buf->dma_dir, &attrs);
if (!sgt->nents)
goto userptr_fail_map;
return buf;
userptr_fail_map:
sg_free_table(&buf->sg_table);
userptr_fail_alloc_table_from_pages:
userptr_fail_get_user_pages:
dprintk(1, "get_user_pages requested/got: %d/%d]\n",
buf->num_pages, num_pages_from_user);
if (!vma_is_io(buf->vma))
while (--num_pages_from_user >= 0)
put_page(buf->pages[num_pages_from_user]);
vb2_put_vma(buf->vma);
userptr_fail_find_vma:
kfree(buf->pages);
userptr_fail_alloc_pages:
kfree(buf);
return NULL;
}
static void *vb2_dma_sg_alloc(void *alloc_ctx, unsigned long size,
enum dma_data_direction dma_dir, gfp_t gfp_flags)
{
struct vb2_dma_sg_conf *conf = alloc_ctx;
struct vb2_dma_sg_buf *buf;
struct sg_table *sgt;
int ret;
int num_pages;
DEFINE_DMA_ATTRS(attrs);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0)
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
#endif
if (WARN_ON(alloc_ctx == NULL))
return NULL;
buf = kzalloc(sizeof *buf, GFP_KERNEL);
if (!buf)
return NULL;
buf->vaddr = NULL;
buf->dma_dir = dma_dir;
buf->offset = 0;
buf->size = size;
/* size is already page aligned */
buf->num_pages = size >> PAGE_SHIFT;
buf->dma_sgt = &buf->sg_table;
buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),
GFP_KERNEL);
if (!buf->pages)
goto fail_pages_array_alloc;
ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags);
if (ret)
goto fail_pages_alloc;
ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
buf->num_pages, 0, size, GFP_KERNEL);
if (ret)
goto fail_table_alloc;
/* Prevent the device from being released while the buffer is used */
buf->dev = get_device(conf->dev);
sgt = &buf->sg_table;
/*
* No need to sync to the device, this will happen later when the
* prepare() memop is called.
*/
sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
buf->dma_dir, &attrs);
if (!sgt->nents)
goto fail_map;
buf->handler.refcount = &buf->refcount;
buf->handler.put = vb2_dma_sg_put;
buf->handler.arg = buf;
atomic_inc(&buf->refcount);
dprintk(1, "%s: Allocated buffer of %d pages\n",
__func__, buf->num_pages);
return buf;
fail_map:
put_device(buf->dev);
sg_free_table(buf->dma_sgt);
fail_table_alloc:
num_pages = buf->num_pages;
while (num_pages--)
__free_page(buf->pages[num_pages]);
fail_pages_alloc:
kfree(buf->pages);
fail_pages_array_alloc:
kfree(buf);
return NULL;
}
static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
{
/* Should only be called during free */
sg_free_table(obj->pages);
kfree(obj->pages);
}
static int
qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req)
{
struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req);
struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
enum dma_data_direction dir_src, dir_dst;
struct scatterlist *sg;
bool diff_dst;
gfp_t gfp;
int ret;
rctx->iv = req->info;
rctx->ivsize = crypto_ablkcipher_ivsize(ablkcipher);
rctx->cryptlen = req->nbytes;
diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;
rctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
if (diff_dst)
rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
else
rctx->dst_nents = rctx->src_nents;
if (rctx->src_nents < 0) {
dev_err(qce->dev, "Invalid numbers of src SG.\n");
return rctx->src_nents;
}
if (rctx->dst_nents < 0) {
dev_err(qce->dev, "Invalid numbers of dst SG.\n");
return -rctx->dst_nents;
}
rctx->dst_nents += 1;
gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp);
if (ret)
return ret;
sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);
sg = qce_sgtable_add(&rctx->dst_tbl, req->dst);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto error_free;
}
sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto error_free;
}
sg_mark_end(sg);
rctx->dst_sg = rctx->dst_tbl.sgl;
ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (ret < 0)
goto error_free;
if (diff_dst) {
ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src);
if (ret < 0)
goto error_unmap_dst;
rctx->src_sg = req->src;
} else {
rctx->src_sg = rctx->dst_sg;
}
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents,
rctx->dst_sg, rctx->dst_nents,
qce_ablkcipher_done, async_req);
if (ret)
goto error_unmap_src;
qce_dma_issue_pending(&qce->dma);
ret = qce_start(async_req, tmpl->crypto_alg_type, req->nbytes, 0);
if (ret)
goto error_terminate;
return 0;
error_terminate:
qce_dma_terminate_all(&qce->dma);
error_unmap_src:
if (diff_dst)
dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src);
error_unmap_dst:
dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
error_free:
sg_free_table(&rctx->dst_tbl);
return ret;
}
