static int
nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
{
struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
struct nouveau_bo *nvbo = nouveau_bo(bo);
/* as long as the bo isn't in vram, and isn't tiled, we've got
* nothing to do here.
*/
if (bo->mem.mem_type != TTM_PL_VRAM) {
if (dev_priv->card_type < NV_50 ||
!nouveau_bo_tile_layout(nvbo))
return 0;
}
/* make sure bo is in mappable vram */
if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
return 0;
nvbo->placement.fpfn = 0;
nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0);
return nouveau_bo_validate(nvbo, false, true, false);
}
static void
nvc0_m2mf_push_rect(struct pipe_screen *pscreen,
const struct nv50_m2mf_rect *dst,
const void *data,
unsigned nblocksx, unsigned nblocksy)
{
struct nouveau_channel *chan;
const uint8_t *src = (const uint8_t *)data;
const int cpp = dst->cpp;
const int line_len = nblocksx * cpp;
int dy = dst->y;
assert(nouveau_bo_tile_layout(dst->bo));
BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5);
OUT_RING (chan, dst->tile_mode);
OUT_RING (chan, dst->width * cpp);
OUT_RING (chan, dst->height);
OUT_RING (chan, dst->depth);
OUT_RING (chan, dst->z);
while (nblocksy) {
int line_count, words;
int size = MIN2(AVAIL_RING(chan), NV04_PFIFO_MAX_PACKET_LEN);
if (size < (12 + words)) {
FIRE_RING(chan);
continue;
}
line_count = (size * 4) / line_len;
words = (line_count * line_len + 3) / 4;
BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
OUT_RELOCh(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR);
OUT_RELOCl(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR);
BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2);
OUT_RING (chan, dst->x * cpp);
OUT_RING (chan, dy);
BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
OUT_RING (chan, line_len);
OUT_RING (chan, line_count);
BEGIN_RING(chan, RING_MF(EXEC), 1);
OUT_RING (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
NVC0_M2MF_EXEC_PUSH | NVC0_M2MF_EXEC_LINEAR_IN);
BEGIN_RING_NI(chan, RING_MF(DATA), words);
OUT_RINGp (chan, src, words);
dy += line_count;
src += line_len * line_count;
nblocksy -= line_count;
}
}
static int
nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
struct nouveau_bo *nvbo = nouveau_bo(bo);
u64 length = (new_mem->num_pages << PAGE_SHIFT);
u64 src_offset, dst_offset;
int ret;
src_offset = old_mem->start << PAGE_SHIFT;
dst_offset = new_mem->start << PAGE_SHIFT;
if (!nvbo->no_vm) {
if (old_mem->mem_type == TTM_PL_VRAM)
src_offset = nvbo->vma.offset;
else
src_offset += dev_priv->gart_info.aper_base;
if (new_mem->mem_type == TTM_PL_VRAM)
dst_offset = nvbo->vma.offset;
else
dst_offset += dev_priv->gart_info.aper_base;
}
ret = RING_SPACE(chan, 3);
if (ret)
return ret;
BEGIN_RING(chan, NvSubM2MF, 0x0184, 2);
OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
while (length) {
u32 amount, stride, height;
amount = min(length, (u64)(4 * 1024 * 1024));
stride = 16 * 4;
height = amount / stride;
if (new_mem->mem_type == TTM_PL_VRAM &&
nouveau_bo_tile_layout(nvbo)) {
ret = RING_SPACE(chan, 8);
if (ret)
return ret;
BEGIN_RING(chan, NvSubM2MF, 0x0200, 7);
OUT_RING (chan, 0);
OUT_RING (chan, 0);
OUT_RING (chan, stride);
OUT_RING (chan, height);
OUT_RING (chan, 1);
OUT_RING (chan, 0);
OUT_RING (chan, 0);
} else {
ret = RING_SPACE(chan, 2);
if (ret)
return ret;
BEGIN_RING(chan, NvSubM2MF, 0x0200, 1);
OUT_RING (chan, 1);
}
if (old_mem->mem_type == TTM_PL_VRAM &&
nouveau_bo_tile_layout(nvbo)) {
ret = RING_SPACE(chan, 8);
if (ret)
return ret;
BEGIN_RING(chan, NvSubM2MF, 0x021c, 7);
OUT_RING (chan, 0);
OUT_RING (chan, 0);
OUT_RING (chan, stride);
OUT_RING (chan, height);
OUT_RING (chan, 1);
OUT_RING (chan, 0);
OUT_RING (chan, 0);
} else {
ret = RING_SPACE(chan, 2);
if (ret)
return ret;
BEGIN_RING(chan, NvSubM2MF, 0x021c, 1);
OUT_RING (chan, 1);
}
ret = RING_SPACE(chan, 14);
if (ret)
return ret;
BEGIN_RING(chan, NvSubM2MF, 0x0238, 2);
OUT_RING (chan, upper_32_bits(src_offset));
OUT_RING (chan, upper_32_bits(dst_offset));
BEGIN_RING(chan, NvSubM2MF, 0x030c, 8);
OUT_RING (chan, lower_32_bits(src_offset));
OUT_RING (chan, lower_32_bits(dst_offset));
OUT_RING (chan, stride);
OUT_RING (chan, stride);
OUT_RING (chan, stride);
OUT_RING (chan, height);
OUT_RING (chan, 0x00000101);
OUT_RING (chan, 0x00000000);
BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
OUT_RING (chan, 0);
length -= amount;
src_offset += amount;
dst_offset += amount;
}
return 0;
}
void
nvc0_m2mf_transfer_rect(struct pipe_screen *pscreen,
const struct nv50_m2mf_rect *dst,
const struct nv50_m2mf_rect *src,
uint32_t nblocksx, uint32_t nblocksy)
{
struct nouveau_channel *chan = nouveau_screen(pscreen)->channel;
const int cpp = dst->cpp;
uint32_t src_ofst = src->base;
uint32_t dst_ofst = dst->base;
uint32_t height = nblocksy;
uint32_t sy = src->y;
uint32_t dy = dst->y;
uint32_t exec = (1 << 20);
assert(dst->cpp == src->cpp);
if (nouveau_bo_tile_layout(src->bo)) {
BEGIN_RING(chan, RING_MF(TILING_MODE_IN), 5);
OUT_RING (chan, src->tile_mode);
OUT_RING (chan, src->width * cpp);
OUT_RING (chan, src->height);
OUT_RING (chan, src->depth);
OUT_RING (chan, src->z);
} else {
src_ofst += src->y * src->pitch + src->x * cpp;
BEGIN_RING(chan, RING_MF(PITCH_IN), 1);
OUT_RING (chan, src->width * cpp);
exec |= NVC0_M2MF_EXEC_LINEAR_IN;
}
if (nouveau_bo_tile_layout(dst->bo)) {
BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5);
OUT_RING (chan, dst->tile_mode);
OUT_RING (chan, dst->width * cpp);
OUT_RING (chan, dst->height);
OUT_RING (chan, dst->depth);
OUT_RING (chan, dst->z);
} else {
dst_ofst += dst->y * dst->pitch + dst->x * cpp;
BEGIN_RING(chan, RING_MF(PITCH_OUT), 1);
OUT_RING (chan, dst->width * cpp);
exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
}
while (height) {
int line_count = height > 2047 ? 2047 : height;
MARK_RING (chan, 17, 4);
BEGIN_RING(chan, RING_MF(OFFSET_IN_HIGH), 2);
OUT_RELOCh(chan, src->bo, src_ofst, src->domain | NOUVEAU_BO_RD);
OUT_RELOCl(chan, src->bo, src_ofst, src->domain | NOUVEAU_BO_RD);
BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
OUT_RELOCh(chan, dst->bo, dst_ofst, dst->domain | NOUVEAU_BO_WR);
OUT_RELOCl(chan, dst->bo, dst_ofst, dst->domain | NOUVEAU_BO_WR);
if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
BEGIN_RING(chan, RING_MF(TILING_POSITION_IN_X), 2);
OUT_RING (chan, src->x * cpp);
OUT_RING (chan, sy);
} else {
src_ofst += line_count * src->pitch;
}
if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2);
OUT_RING (chan, dst->x * cpp);
OUT_RING (chan, dy);
} else {
dst_ofst += line_count * dst->pitch;
}
BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
OUT_RING (chan, nblocksx * cpp);
OUT_RING (chan, line_count);
BEGIN_RING(chan, RING_MF(EXEC), 1);
OUT_RING (chan, exec);
height -= line_count;
sy += line_count;
dy += line_count;
}
}