int
pscnv_mem_free(struct pscnv_bo *bo)
{
struct drm_nouveau_private *dev_priv = bo->dev->dev_private;
if (pscnv_mem_debug >= 1)
NV_INFO(bo->dev, "Freeing %d, %#llx-byte %sBO(%p) of type %08x, tile_flags %x\n", bo->serial, bo->size,
(bo->flags & PSCNV_GEM_CONTIG ? "contig " : ""), bo, bo->cookie, bo->tile_flags);
if (dev_priv->vm_ok && bo->map1)
pscnv_vspace_unmap_node(bo->map1);
if (dev_priv->vm_ok && bo->map3)
pscnv_vspace_unmap_node(bo->map3);
switch (bo->flags & PSCNV_GEM_MEMTYPE_MASK) {
case PSCNV_GEM_VRAM_SMALL:
case PSCNV_GEM_VRAM_LARGE:
dev_priv->vram->free(bo);
break;
case PSCNV_GEM_SYSRAM_SNOOP:
case PSCNV_GEM_SYSRAM_NOSNOOP:
pscnv_sysram_free(bo);
break;
}
kfree (bo);
return 0;
}
int
pscnv_dma_chunk_to_chunk(struct pscnv_chunk *from, struct pscnv_chunk *to, int flags)
{
struct drm_device *dev = from->bo->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
struct pscnv_dma *dma = dev_priv->dma;
struct pscnv_client *client = from->bo->client;
struct pscnv_mm_node *from_node;
struct pscnv_mm_node *to_node;
const uint64_t size = pscnv_chunk_size(from);
struct timespec start, start_real, end, end_real;
s64 start_ns, start_real_ns, duration, duration_real;
int ret;
flags = pscnv_dma_setup_flags(flags);
if (!dma) {
NV_ERROR(dev, "DMA: not available\n");
return -ENOENT;
}
BUG_ON(from->bo->dev != to->bo->dev);
mutex_lock(&dma->lock);
getnstimeofday(&start_real);
start_real_ns = timespec_to_ns(&start_real);
if (pscnv_chunk_size(to) < size) {
NV_INFO(dev, "DMA: source chunk %08x/%d-%u has size %lld, but "
"target chunk %08x/%d-%u only has size %lld\n",
from->bo->cookie, from->bo->serial, from->idx,
size, to->bo->cookie, to->bo->serial, to->idx,
pscnv_chunk_size(to));
return -ENOSPC;
}
ret = pscnv_vspace_map_chunk(dma->vs, to,
0x20000000, /* start */
1ull << 40, /* end */
0, /* back, nonsense? */
&to_node);
if (ret) {
NV_INFO(dev, "DMA: failed to map 'to' chunk\n");
goto fail_map_to;
}
ret = pscnv_vspace_map_chunk(dma->vs, from,
0x20000000, /* start */
1ull << 40, /* end */
0, /* back, nonsense? */
&from_node);
if (ret) {
NV_INFO(dev, "DMA: failed to map 'from' chunk\n");
goto fail_map_from;
}
if (flags & PSCNV_DMA_DEBUG) {
dev_priv->chan->pd_dump_chan(dev, NULL /* seq_file */, PSCNV_DMA_CHAN);
}
getnstimeofday(&start);
start_ns = timespec_to_ns(&start);
pscnv_ib_membar(dma->ib_chan);
if (flags & PSCNV_DMA_ASYNC) {
pscnv_ib_fence_write(dma->ib_chan, GDEV_SUBCH_NV_PCOPY0);
nvc0_memcpy_pcopy0(dma->ib_chan, to_node->start, from_node->start, size, flags);
} else {
pscnv_ib_fence_write(dma->ib_chan, GDEV_SUBCH_NV_M2MF);
nvc0_memcpy_m2mf(dma->ib_chan, to_node->start, from_node->start, size, flags);
}
ret = pscnv_ib_fence_wait(dma->ib_chan);
if (ret) {
NV_INFO(dev, "DMA: failed to wait for fence completion\n");
goto fail_fence_wait;
}
getnstimeofday(&end);
/* no return here, always unmap memory */
fail_fence_wait:
pscnv_vspace_unmap_node(from_node);
fail_map_from:
pscnv_vspace_unmap_node(to_node);
fail_map_to:
mutex_unlock(&dma->lock);
getnstimeofday(&end_real);
if (!ret) {
duration = timespec_to_ns(&end) - start_ns;
duration_real = timespec_to_ns(&end_real) - start_real_ns;
if (flags & PSCNV_DMA_VERBOSE) {
NV_INFO(dev, "DMA: took %lld.%04lld ms (real %lld.%04lld ms)\n",
duration / 1000000,
(duration % 1000000) / 100,
duration_real / 1000000,
(duration_real % 1000000) / 100);
}
pscnv_client_track_time(client, start_ns, duration, size, "DMA");
pscnv_client_track_time(client, start_ns, duration_real, size, "DMA_REAL");
if (client)
client->pause_bytes_transferred += size;
}
return ret;
}
int
pscnv_dma_bo_to_bo(struct pscnv_bo *tgt, struct pscnv_bo *src, int flags) {
struct drm_device *dev = tgt->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
struct pscnv_dma *dma = dev_priv->dma;
struct pscnv_mm_node *tgt_node;
struct pscnv_mm_node *src_node;
const uint32_t size = tgt->size;
struct timespec start, start_real, end, end_real;
s64 start_ns, start_real_ns, duration, duration_real;
int ret;
flags = pscnv_dma_setup_flags(flags);
if (!dma) {
NV_ERROR(dev, "DMA: not available\n");
return -ENOENT;
}
BUG_ON(tgt->dev != src->dev);
mutex_lock(&dma->lock);
getnstimeofday(&start_real);
start_real_ns = timespec_to_ns(&start_real);
if (tgt->size < src->size) {
NV_INFO(dev, "DMA: source bo (cookie=%x) has size %lld, but target bo "
"(cookie=%x) only has size %lld\n",
src->cookie, src->size, tgt->cookie, tgt->size);
return -ENOSPC;
}
ret = pscnv_vspace_map(dma->vs, tgt,
0x20000000, /* start */
1ull << 40, /* end */
0, /* back, nonsense? */
&tgt_node);
if (ret) {
NV_INFO(dev, "DMA: failed to map target bo\n");
goto fail_map_tgt;
}
ret = pscnv_vspace_map(dma->vs, src,
0x20000000, /* start */
1ull << 40, /* end */
0, /* back, nonsense? */
&src_node);
if (ret) {
NV_INFO(dev, "DMA: failed to map source bo\n");
goto fail_map_src;
}
if (flags & PSCNV_DMA_DEBUG) {
dev_priv->chan->pd_dump_chan(dev, NULL /* seq_file */, PSCNV_DMA_CHAN);
}
getnstimeofday(&start);
start_ns = timespec_to_ns(&start);
pscnv_ib_membar(dma->ib_chan);
if (flags & PSCNV_DMA_ASYNC) {
pscnv_ib_fence_write(dma->ib_chan, GDEV_SUBCH_NV_PCOPY0);
nvc0_memcpy_pcopy0(dma->ib_chan, tgt_node->start, src_node->start, size, flags);
} else {
pscnv_ib_fence_write(dma->ib_chan, GDEV_SUBCH_NV_M2MF);
nvc0_memcpy_m2mf(dma->ib_chan, tgt_node->start, src_node->start, size, flags);
}
ret = pscnv_ib_fence_wait(dma->ib_chan);
if (ret) {
NV_INFO(dev, "DMA: failed to wait for fence completion\n");
goto fail_fence_wait;
}
getnstimeofday(&end);
/* no return here, always unmap memory */
fail_fence_wait:
pscnv_vspace_unmap_node(src_node);
fail_map_src:
pscnv_vspace_unmap_node(tgt_node);
fail_map_tgt:
mutex_unlock(&dma->lock);
getnstimeofday(&end_real);
if (!ret) {
duration = timespec_to_ns(&end) - start_ns;
duration_real = timespec_to_ns(&end_real) - start_real_ns;
if (flags & PSCNV_DMA_VERBOSE) {
NV_INFO(dev, "DMA: took %lld.%04lld ms (real %lld.%04lld ms)\n",
duration / 1000000,
(duration % 1000000) / 100,
duration_real / 1000000,
(duration_real % 1000000) / 100);
}
pscnv_client_track_time(src->client, start_ns, duration, size, "DMA");
pscnv_client_track_time(src->client, start_ns, duration_real, size, "DMA_REAL");
if (src->client)
src->client->pause_bytes_transferred += size;
}
return ret;
}
