void
nvc0_fragprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *fp = nvc0->fragprog;
if (!nvc0_program_validate(nvc0, fp))
return;
nvc0_program_update_context_state(nvc0, fp, 4);
if (fp->fp.early_z != nvc0->state.early_z_forced) {
nvc0->state.early_z_forced = fp->fp.early_z;
IMMED_NVC0(push, NVC0_3D(FORCE_EARLY_FRAGMENT_TESTS), fp->fp.early_z);
}
BEGIN_NVC0(push, NVC0_3D(SP_SELECT(5)), 2);
PUSH_DATA (push, 0x51);
PUSH_DATA (push, fp->code_base);
BEGIN_NVC0(push, NVC0_3D(SP_GPR_ALLOC(5)), 1);
PUSH_DATA (push, fp->max_gpr);
BEGIN_NVC0(push, SUBC_3D(0x0360), 2);
PUSH_DATA (push, 0x20164010);
PUSH_DATA (push, 0x20);
BEGIN_NVC0(push, NVC0_3D(ZCULL_TEST_MASK), 1);
PUSH_DATA (push, fp->flags[0]);
}
static void
nve4_m2mf_copy_linear(struct nouveau_context *nv,
struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
unsigned size)
{
struct nouveau_pushbuf *push = nv->pushbuf;
struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
nouveau_pushbuf_bufctx(push, bctx);
nouveau_pushbuf_validate(push);
BEGIN_NVC0(push, SUBC_COPY(0x0400), 4);
PUSH_DATAh(push, src->offset + srcoff);
PUSH_DATA (push, src->offset + srcoff);
PUSH_DATAh(push, dst->offset + dstoff);
PUSH_DATA (push, dst->offset + dstoff);
BEGIN_NVC0(push, SUBC_COPY(0x0418), 1);
PUSH_DATA (push, size);
BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
PUSH_DATA (push, 0x186);
nouveau_bufctx_reset(bctx, 0);
}
void
nvc0_tctlprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *tp = nvc0->tctlprog;
if (tp && nvc0_program_validate(nvc0, tp)) {
if (tp->tp.tess_mode != ~0) {
BEGIN_NVC0(push, NVC0_3D(TESS_MODE), 1);
PUSH_DATA (push, tp->tp.tess_mode);
}
BEGIN_NVC0(push, NVC0_3D(SP_SELECT(2)), 2);
PUSH_DATA (push, 0x21);
PUSH_DATA (push, tp->code_base);
BEGIN_NVC0(push, NVC0_3D(SP_GPR_ALLOC(2)), 1);
PUSH_DATA (push, tp->max_gpr);
if (tp->tp.input_patch_size <= 32)
IMMED_NVC0(push, NVC0_3D(PATCH_VERTICES), tp->tp.input_patch_size);
} else {
BEGIN_NVC0(push, NVC0_3D(SP_SELECT(2)), 1);
PUSH_DATA (push, 0x20);
}
nvc0_program_update_context_state(nvc0, tp, 1);
}
/* Upload the "diagonal" entries for the possible texture sources ($t == $s).
* At some point we might want to get a list of the combinations used by a
* shader and fill in those entries instead of having it extract the handles.
*/
void
nve4_set_tex_handles(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
uint64_t address;
unsigned s;
if (nvc0->screen->base.class_3d < NVE4_3D_CLASS)
return;
address = nvc0->screen->uniform_bo->offset + (5 << 16);
for (s = 0; s < 5; ++s, address += (1 << 9)) {
uint32_t dirty = nvc0->textures_dirty[s] | nvc0->samplers_dirty[s];
if (!dirty)
continue;
BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
PUSH_DATA (push, 512);
PUSH_DATAh(push, address);
PUSH_DATA (push, address);
do {
int i = ffs(dirty) - 1;
dirty &= ~(1 << i);
BEGIN_NVC0(push, NVC0_3D(CB_POS), 2);
PUSH_DATA (push, (8 + i) * 4);
PUSH_DATA (push, nvc0->tex_handles[s][i]);
} while (dirty);
nvc0->textures_dirty[s] = 0;
nvc0->samplers_dirty[s] = 0;
}
}
static INLINE void
nvc0_program_update_context_state(struct nvc0_context *nvc0,
struct nvc0_program *prog, int stage)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
if (prog && prog->need_tls) {
const uint32_t flags = NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR;
if (!nvc0->state.tls_required)
BCTX_REFN_bo(nvc0->bufctx_3d, TLS, flags, nvc0->screen->tls);
nvc0->state.tls_required |= 1 << stage;
} else {
if (nvc0->state.tls_required == (1 << stage))
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_TLS);
nvc0->state.tls_required &= ~(1 << stage);
}
if (prog && prog->immd_size) {
BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
/* NOTE: may overlap code of a different shader */
PUSH_DATA (push, align(prog->immd_size, 0x100));
PUSH_DATAh(push, nvc0->screen->text->offset + prog->immd_base);
PUSH_DATA (push, nvc0->screen->text->offset + prog->immd_base);
BEGIN_NVC0(push, NVC0_3D(CB_BIND(stage)), 1);
PUSH_DATA (push, (14 << 4) | 1);
nvc0->state.c14_bound |= 1 << stage;
} else
if (nvc0->state.c14_bound & (1 << stage)) {
BEGIN_NVC0(push, NVC0_3D(CB_BIND(stage)), 1);
PUSH_DATA (push, (14 << 4) | 0);
nvc0->state.c14_bound &= ~(1 << stage);
}
}
void
nvc0_gmtyprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *gp = nvc0->gmtyprog;
if (gp)
nvc0_program_validate(nvc0, gp);
/* we allow GPs with no code for specifying stream output state only */
if (gp && gp->code_size) {
const boolean gp_selects_layer = gp->hdr[13] & (1 << 9);
BEGIN_NVC0(push, NVC0_3D(MACRO_GP_SELECT), 1);
PUSH_DATA (push, 0x41);
BEGIN_NVC0(push, NVC0_3D(SP_START_ID(4)), 1);
PUSH_DATA (push, gp->code_base);
BEGIN_NVC0(push, NVC0_3D(SP_GPR_ALLOC(4)), 1);
PUSH_DATA (push, gp->max_gpr);
BEGIN_NVC0(push, NVC0_3D(LAYER), 1);
PUSH_DATA (push, gp_selects_layer ? NVC0_3D_LAYER_USE_GP : 0);
} else {
IMMED_NVC0(push, NVC0_3D(LAYER), 0);
BEGIN_NVC0(push, NVC0_3D(MACRO_GP_SELECT), 1);
PUSH_DATA (push, 0x40);
}
nvc0_program_update_context_state(nvc0, gp, 3);
}
void
nvc0_tctlprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *tp = nvc0->tctlprog;
if (tp && nvc0_program_validate(nvc0, tp)) {
if (tp->tp.tess_mode != ~0) {
BEGIN_NVC0(push, NVC0_3D(TESS_MODE), 1);
PUSH_DATA (push, tp->tp.tess_mode);
}
BEGIN_NVC0(push, NVC0_3D(SP_SELECT(2)), 2);
PUSH_DATA (push, 0x21);
PUSH_DATA (push, tp->code_base);
BEGIN_NVC0(push, NVC0_3D(SP_GPR_ALLOC(2)), 1);
PUSH_DATA (push, tp->num_gprs);
} else {
tp = nvc0->tcp_empty;
/* not a whole lot we can do to handle this failure */
if (!nvc0_program_validate(nvc0, tp))
assert(!"unable to validate empty tcp");
BEGIN_NVC0(push, NVC0_3D(SP_SELECT(2)), 2);
PUSH_DATA (push, 0x20);
PUSH_DATA (push, tp->code_base);
}
nvc0_program_update_context_state(nvc0, tp, 1);
}
static boolean
nve4_validate_tic(struct nvc0_context *nvc0, unsigned s)
{
struct nouveau_bo *txc = nvc0->screen->txc;
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
unsigned i;
boolean need_flush = FALSE;
for (i = 0; i < nvc0->num_textures[s]; ++i) {
struct nv50_tic_entry *tic = nv50_tic_entry(nvc0->textures[s][i]);
struct nv04_resource *res;
const boolean dirty = !!(nvc0->textures_dirty[s] & (1 << i));
if (!tic) {
nvc0->tex_handles[s][i] |= NVE4_TIC_ENTRY_INVALID;
continue;
}
res = nv04_resource(tic->pipe.texture);
if (tic->id < 0) {
tic->id = nvc0_screen_tic_alloc(nvc0->screen, tic);
PUSH_SPACE(push, 16);
BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
PUSH_DATAh(push, txc->offset + (tic->id * 32));
PUSH_DATA (push, txc->offset + (tic->id * 32));
BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
PUSH_DATA (push, 32);
PUSH_DATA (push, 1);
BEGIN_1IC0(push, NVE4_P2MF(EXEC), 9);
PUSH_DATA (push, 0x1001);
PUSH_DATAp(push, &tic->tic[0], 8);
need_flush = TRUE;
} else
if (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
BEGIN_NVC0(push, NVC0_3D(TEX_CACHE_CTL), 1);
PUSH_DATA (push, (tic->id << 4) | 1);
}
nvc0->screen->tic.lock[tic->id / 32] |= 1 << (tic->id % 32);
res->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
res->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
nvc0->tex_handles[s][i] &= ~NVE4_TIC_ENTRY_INVALID;
nvc0->tex_handles[s][i] |= tic->id;
if (dirty)
BCTX_REFN(nvc0->bufctx_3d, TEX(s, i), res, RD);
}
for (; i < nvc0->state.num_textures[s]; ++i) {
nvc0->tex_handles[s][i] |= NVE4_TIC_ENTRY_INVALID;
nvc0->textures_dirty[s] |= 1 << i;
}
nvc0->state.num_textures[s] = nvc0->num_textures[s];
return need_flush;
}
static int
nouveau_fbcon_sync(struct fb_info *info)
{
struct nouveau_fbdev *nfbdev = info->par;
struct drm_device *dev = nfbdev->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
struct nouveau_channel *chan = dev_priv->channel;
int ret, i;
if (!chan || !chan->accel_done || in_interrupt() ||
info->state != FBINFO_STATE_RUNNING ||
info->flags & FBINFO_HWACCEL_DISABLED)
return 0;
if (!mutex_trylock(&chan->mutex))
return 0;
ret = RING_SPACE(chan, 4);
if (ret) {
mutex_unlock(&chan->mutex);
nouveau_fbcon_gpu_lockup(info);
return 0;
}
if (dev_priv->card_type >= NV_C0) {
BEGIN_NVC0(chan, 2, NvSub2D, 0x010c, 1);
OUT_RING (chan, 0);
BEGIN_NVC0(chan, 2, NvSub2D, 0x0100, 1);
OUT_RING (chan, 0);
} else {
BEGIN_RING(chan, 0, 0x0104, 1);
OUT_RING (chan, 0);
BEGIN_RING(chan, 0, 0x0100, 1);
OUT_RING (chan, 0);
}
nouveau_bo_wr32(chan->notifier_bo, chan->m2mf_ntfy/4 + 3, 0xffffffff);
FIRE_RING(chan);
mutex_unlock(&chan->mutex);
ret = -EBUSY;
for (i = 0; i < 100000; i++) {
if (!nouveau_bo_rd32(chan->notifier_bo, chan->m2mf_ntfy/4 + 3)) {
ret = 0;
break;
}
DRM_UDELAY(1);
}
if (ret) {
nouveau_fbcon_gpu_lockup(info);
return 0;
}
chan->accel_done = false;
return 0;
}
static void
disp_vertices_i32(struct push_context *ctx, unsigned start, unsigned count)
{
struct nouveau_pushbuf *push = ctx->push;
struct translate *translate = ctx->translate;
const uint32_t *restrict elts = (uint32_t *)ctx->idxbuf + start;
unsigned pos = 0;
do {
unsigned nR = count;
if (unlikely(ctx->prim_restart))
nR = prim_restart_search_i32(elts, nR, ctx->restart_index);
translate->run_elts(translate, elts, nR, 0, ctx->instance_id, ctx->dest);
count -= nR;
ctx->dest += nR * ctx->vertex_size;
while (nR) {
unsigned nE = nR;
if (unlikely(ctx->edgeflag.enabled))
nE = ef_toggle_search_i32(ctx, elts, nR);
PUSH_SPACE(push, 4);
if (likely(nE >= 2)) {
BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
PUSH_DATA (push, pos);
PUSH_DATA (push, nE);
} else
if (nE) {
if (pos <= 0xff) {
IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
} else {
BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
PUSH_DATA (push, pos);
}
}
if (unlikely(nE != nR))
IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
pos += nE;
elts += nE;
nR -= nE;
}
if (count) {
BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
PUSH_DATA (push, ctx->restart_index);
++elts;
ctx->dest += ctx->vertex_size;
++pos;
--count;
}
} while (count);
}
static int
nvc0_2d_texture_do_copy(struct nouveau_pushbuf *push,
struct nv50_miptree *dst, unsigned dst_level,
unsigned dx, unsigned dy, unsigned dz,
struct nv50_miptree *src, unsigned src_level,
unsigned sx, unsigned sy, unsigned sz,
unsigned w, unsigned h)
{
static const uint32_t duvdxy[5] =
{
0x40000000, 0x80000000, 0x00000001, 0x00000002, 0x00000004
};
int ret;
uint32_t ctrl = 0x00;
ret = PUSH_SPACE(push, 2 * 16 + 32);
if (ret)
return ret;
ret = nvc0_2d_texture_set(push, TRUE, dst, dst_level, dz);
if (ret)
return ret;
ret = nvc0_2d_texture_set(push, FALSE, src, src_level, sz);
if (ret)
return ret;
/* NOTE: 2D engine doesn't work for MS8 */
if (src->ms_x)
ctrl = 0x11;
/* 0/1 = CENTER/CORNER, 00/10 = POINT/BILINEAR */
BEGIN_NVC0(push, NVC0_2D(BLIT_CONTROL), 1);
PUSH_DATA (push, ctrl);
BEGIN_NVC0(push, NVC0_2D(BLIT_DST_X), 4);
PUSH_DATA (push, dx << dst->ms_x);
PUSH_DATA (push, dy << dst->ms_y);
PUSH_DATA (push, w << dst->ms_x);
PUSH_DATA (push, h << dst->ms_y);
BEGIN_NVC0(push, NVC0_2D(BLIT_DU_DX_FRACT), 4);
PUSH_DATA (push, duvdxy[2 + ((int)src->ms_x - (int)dst->ms_x)] & 0xf0000000);
PUSH_DATA (push, duvdxy[2 + ((int)src->ms_x - (int)dst->ms_x)] & 0x0000000f);
PUSH_DATA (push, duvdxy[2 + ((int)src->ms_y - (int)dst->ms_y)] & 0xf0000000);
PUSH_DATA (push, duvdxy[2 + ((int)src->ms_y - (int)dst->ms_y)] & 0x0000000f);
BEGIN_NVC0(push, NVC0_2D(BLIT_SRC_X_FRACT), 4);
PUSH_DATA (push, 0);
PUSH_DATA (push, sx << src->ms_x);
PUSH_DATA (push, 0);
PUSH_DATA (push, sy << src->ms_x);
return 0;
}
static int
nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
struct nouveau_bo *nvbo = nouveau_bo(bo);
u64 src_offset = old_mem->start << PAGE_SHIFT;
u64 dst_offset = new_mem->start << PAGE_SHIFT;
u32 page_count = new_mem->num_pages;
int ret;
if (!nvbo->no_vm) {
if (old_mem->mem_type == TTM_PL_VRAM)
src_offset = nvbo->vma.offset;
else
src_offset += dev_priv->gart_info.aper_base;
if (new_mem->mem_type == TTM_PL_VRAM)
dst_offset = nvbo->vma.offset;
else
dst_offset += dev_priv->gart_info.aper_base;
}
page_count = new_mem->num_pages;
while (page_count) {
int line_count = (page_count > 2047) ? 2047 : page_count;
ret = RING_SPACE(chan, 12);
if (ret)
return ret;
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2);
OUT_RING (chan, upper_32_bits(dst_offset));
OUT_RING (chan, lower_32_bits(dst_offset));
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6);
OUT_RING (chan, upper_32_bits(src_offset));
OUT_RING (chan, lower_32_bits(src_offset));
OUT_RING (chan, PAGE_SIZE); /* src_pitch */
OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
OUT_RING (chan, PAGE_SIZE); /* line_length */
OUT_RING (chan, line_count);
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1);
OUT_RING (chan, 0x00100110);
page_count -= line_count;
src_offset += (PAGE_SIZE * line_count);
dst_offset += (PAGE_SIZE * line_count);
}
return 0;
}
static void
nvc0_compute_validate_driverconst(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_screen *screen = nvc0->screen;
BEGIN_NVC0(push, NVC0_CP(CB_SIZE), 3);
PUSH_DATA (push, NVC0_CB_AUX_SIZE);
PUSH_DATAh(push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(5));
PUSH_DATA (push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(5));
BEGIN_NVC0(push, NVC0_CP(CB_BIND), 1);
PUSH_DATA (push, (15 << 8) | 1);
nvc0->dirty_3d |= NVC0_NEW_3D_DRIVERCONST;
}
boolean
nvc0_compute_validate_program(struct nvc0_context *nvc0)
{
struct nvc0_program *prog = nvc0->compprog;
if (prog->mem)
return TRUE;
if (!prog->translated) {
prog->translated = nvc0_program_translate(
prog, nvc0->screen->base.device->chipset);
if (!prog->translated)
return FALSE;
}
if (unlikely(!prog->code_size))
return FALSE;
if (likely(prog->code_size)) {
if (nvc0_program_upload_code(nvc0, prog)) {
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
BEGIN_NVC0(push, NVC0_COMPUTE(FLUSH), 1);
PUSH_DATA (push, NVC0_COMPUTE_FLUSH_CODE);
return TRUE;
}
}
return FALSE;
}
/* This happens rather often with DTD9/st. */
void
nvc0_cb_push(struct nouveau_context *nv,
struct nouveau_bo *bo, unsigned domain,
unsigned base, unsigned size,
unsigned offset, unsigned words, const uint32_t *data)
{
struct nouveau_pushbuf *push = nv->pushbuf;
NOUVEAU_DRV_STAT(nv->screen, constbuf_upload_count, 1);
NOUVEAU_DRV_STAT(nv->screen, constbuf_upload_bytes, words * 4);
assert(!(offset & 3));
size = align(size, 0x100);
BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
PUSH_DATA (push, size);
PUSH_DATAh(push, bo->offset + base);
PUSH_DATA (push, bo->offset + base);
while (words) {
unsigned nr = PUSH_AVAIL(push);
nr = MIN2(nr, words);
nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
PUSH_SPACE(push, nr + 2);
PUSH_REFN (push, bo, NOUVEAU_BO_WR | domain);
BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
PUSH_DATA (push, offset);
PUSH_DATAp(push, data, nr);
words -= nr;
data += nr;
offset += nr * 4;
}
}
static void
nvc0_render_condition(struct pipe_context *pipe,
struct pipe_query *pq,
boolean condition, uint mode)
{
struct nvc0_context *nvc0 = nvc0_context(pipe);
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_query *q;
uint32_t cond;
boolean negated = FALSE;
boolean wait =
mode != PIPE_RENDER_COND_NO_WAIT &&
mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT;
nvc0->cond_query = pq;
nvc0->cond_cond = condition;
nvc0->cond_mode = mode;
if (!pq) {
PUSH_SPACE(push, 1);
IMMED_NVC0(push, NVC0_3D(COND_MODE), NVC0_3D_COND_MODE_ALWAYS);
return;
}
q = nvc0_query(pq);
/* NOTE: comparison of 2 queries only works if both have completed */
switch (q->type) {
case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
cond = negated ? NVC0_3D_COND_MODE_EQUAL :
NVC0_3D_COND_MODE_NOT_EQUAL;
wait = TRUE;
break;
case PIPE_QUERY_OCCLUSION_COUNTER:
case PIPE_QUERY_OCCLUSION_PREDICATE:
if (likely(!negated)) {
if (unlikely(q->nesting))
cond = wait ? NVC0_3D_COND_MODE_NOT_EQUAL :
NVC0_3D_COND_MODE_ALWAYS;
else
cond = NVC0_3D_COND_MODE_RES_NON_ZERO;
} else {
cond = wait ? NVC0_3D_COND_MODE_EQUAL : NVC0_3D_COND_MODE_ALWAYS;
}
break;
default:
assert(!"render condition query not a predicate");
mode = NVC0_3D_COND_MODE_ALWAYS;
break;
}
if (wait)
nvc0_query_fifo_wait(push, pq);
PUSH_SPACE(push, 4);
PUSH_REFN (push, q->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
BEGIN_NVC0(push, NVC0_3D(COND_ADDRESS_HIGH), 3);
PUSH_DATAh(push, q->bo->offset + q->offset);
PUSH_DATA (push, q->bo->offset + q->offset);
PUSH_DATA (push, cond);
}
static void
disp_vertices_seq(struct push_context *ctx, unsigned start, unsigned count)
{
struct nouveau_pushbuf *push = ctx->push;
struct translate *translate = ctx->translate;
unsigned pos = 0;
translate->run(translate, start, count, 0, ctx->instance_id, ctx->dest);
do {
unsigned nr = count;
if (unlikely(ctx->edgeflag.enabled))
nr = ef_toggle_search_seq(ctx, start + pos, nr);
PUSH_SPACE(push, 4);
if (likely(nr)) {
BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
PUSH_DATA (push, pos);
PUSH_DATA (push, nr);
}
if (unlikely(nr != count))
IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
pos += nr;
count -= nr;
} while (count);
}
static int
nouveau_card_channel_init(struct drm_device *dev)
{
struct drm_nouveau_private *dev_priv = dev->dev_private;
struct nouveau_channel *chan;
int ret, oclass;
ret = nouveau_channel_alloc(dev, &chan, NULL, NvDmaFB, NvDmaTT);
dev_priv->channel = chan;
if (ret)
return ret;
mutex_unlock(&dev_priv->channel->mutex);
if (dev_priv->card_type <= NV_50) {
if (dev_priv->card_type < NV_50)
oclass = 0x0039;
else
oclass = 0x5039;
ret = nouveau_gpuobj_gr_new(chan, NvM2MF, oclass);
if (ret)
goto error;
ret = nouveau_notifier_alloc(chan, NvNotify0, 32, 0xfe0, 0x1000,
&chan->m2mf_ntfy);
if (ret)
goto error;
ret = RING_SPACE(chan, 6);
if (ret)
goto error;
BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NAME, 1);
OUT_RING (chan, NvM2MF);
BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_NOTIFY, 3);
OUT_RING (chan, NvNotify0);
OUT_RING (chan, chan->vram_handle);
OUT_RING (chan, chan->gart_handle);
} else
if (dev_priv->card_type <= NV_D0) {
ret = nouveau_gpuobj_gr_new(chan, 0x9039, 0x9039);
if (ret)
goto error;
ret = RING_SPACE(chan, 2);
if (ret)
goto error;
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0000, 1);
OUT_RING (chan, 0x00009039);
}
FIRE_RING (chan);
error:
if (ret)
nouveau_card_channel_fini(dev);
return ret;
}
static void
nvc0_clear_render_target(struct pipe_context *pipe,
struct pipe_surface *dst,
const union pipe_color_union *color,
unsigned dstx, unsigned dsty,
unsigned width, unsigned height)
{
struct nvc0_context *nvc0 = nvc0_context(pipe);
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nv50_surface *sf = nv50_surface(dst);
struct nv04_resource *res = nv04_resource(sf->base.texture);
unsigned z;
if (!PUSH_SPACE(push, 32 + sf->depth))
return;
PUSH_REFN (push, res->bo, res->domain | NOUVEAU_BO_WR);
BEGIN_NVC0(push, NVC0_3D(CLEAR_COLOR(0)), 4);
PUSH_DATAf(push, color->f[0]);
PUSH_DATAf(push, color->f[1]);
PUSH_DATAf(push, color->f[2]);
PUSH_DATAf(push, color->f[3]);
BEGIN_NVC0(push, NVC0_3D(SCREEN_SCISSOR_HORIZ), 2);
PUSH_DATA (push, ( width << 16) | dstx);
PUSH_DATA (push, (height << 16) | dsty);
BEGIN_NVC0(push, NVC0_3D(RT_CONTROL), 1);
PUSH_DATA (push, 1);
BEGIN_NVC0(push, NVC0_3D(RT_ADDRESS_HIGH(0)), 9);
PUSH_DATAh(push, res->address + sf->offset);
PUSH_DATA (push, res->address + sf->offset);
if (likely(nouveau_bo_memtype(res->bo))) {
struct nv50_miptree *mt = nv50_miptree(dst->texture);
PUSH_DATA(push, sf->width);
PUSH_DATA(push, sf->height);
PUSH_DATA(push, nvc0_format_table[dst->format].rt);
PUSH_DATA(push, (mt->layout_3d << 16) |
mt->level[sf->base.u.tex.level].tile_mode);
PUSH_DATA(push, dst->u.tex.first_layer + sf->depth);
PUSH_DATA(push, mt->layer_stride >> 2);
PUSH_DATA(push, dst->u.tex.first_layer);
} else {
if (res->base.target == PIPE_BUFFER) {
void
nvc0_m2mf_push_linear(struct nouveau_context *nv,
struct nouveau_bo *dst, unsigned offset, unsigned domain,
unsigned size, const void *data)
{
struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
struct nouveau_pushbuf *push = nv->pushbuf;
uint32_t *src = (uint32_t *)data;
unsigned count = (size + 3) / 4;
nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
nouveau_pushbuf_bufctx(push, nvc0->bufctx);
nouveau_pushbuf_validate(push);
while (count) {
unsigned nr;
if (!PUSH_SPACE(push, 16))
break;
nr = PUSH_AVAIL(push);
assert(nr >= 16);
nr = MIN2(count, nr - 9);
nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
PUSH_DATAh(push, dst->offset + offset);
PUSH_DATA (push, dst->offset + offset);
BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
PUSH_DATA (push, MIN2(size, nr * 4));
PUSH_DATA (push, 1);
BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
PUSH_DATA (push, 0x100111);
/* must not be interrupted (trap on QUERY fence, 0x50 works however) */
BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
PUSH_DATAp(push, src, nr);
count -= nr;
src += nr;
offset += nr * 4;
size -= nr * 4;
}
nouveau_bufctx_reset(nvc0->bufctx, 0);
}
boolean
nve4_validate_tsc(struct nvc0_context *nvc0, int s)
{
struct nouveau_bo *txc = nvc0->screen->txc;
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
unsigned i;
boolean need_flush = FALSE;
for (i = 0; i < nvc0->num_samplers[s]; ++i) {
struct nv50_tsc_entry *tsc = nv50_tsc_entry(nvc0->samplers[s][i]);
if (!tsc) {
nvc0->tex_handles[s][i] |= NVE4_TSC_ENTRY_INVALID;
continue;
}
if (tsc->id < 0) {
tsc->id = nvc0_screen_tsc_alloc(nvc0->screen, tsc);
PUSH_SPACE(push, 16);
BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
PUSH_DATAh(push, txc->offset + 65536 + (tsc->id * 32));
PUSH_DATA (push, txc->offset + 65536 + (tsc->id * 32));
BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
PUSH_DATA (push, 32);
PUSH_DATA (push, 1);
BEGIN_1IC0(push, NVE4_P2MF(EXEC), 9);
PUSH_DATA (push, 0x1001);
PUSH_DATAp(push, &tsc->tsc[0], 8);
need_flush = TRUE;
}
nvc0->screen->tsc.lock[tsc->id / 32] |= 1 << (tsc->id % 32);
nvc0->tex_handles[s][i] &= ~NVE4_TSC_ENTRY_INVALID;
nvc0->tex_handles[s][i] |= tsc->id << 20;
}
for (; i < nvc0->state.num_samplers[s]; ++i) {
nvc0->tex_handles[s][i] |= NVE4_TSC_ENTRY_INVALID;
nvc0->samplers_dirty[s] |= 1 << i;
}
nvc0->state.num_samplers[s] = nvc0->num_samplers[s];
return need_flush;
}
static int
nvc0_2d_texture_do_copy(struct nouveau_pushbuf *push,
struct nv50_miptree *dst, unsigned dst_level,
unsigned dx, unsigned dy, unsigned dz,
struct nv50_miptree *src, unsigned src_level,
unsigned sx, unsigned sy, unsigned sz,
unsigned w, unsigned h)
{
const enum pipe_format dfmt = dst->base.base.format;
const enum pipe_format sfmt = src->base.base.format;
int ret;
boolean eqfmt = dfmt == sfmt;
if (!PUSH_SPACE(push, 2 * 16 + 32))
return PIPE_ERROR;
ret = nvc0_2d_texture_set(push, TRUE, dst, dst_level, dz, dfmt, eqfmt);
if (ret)
return ret;
ret = nvc0_2d_texture_set(push, FALSE, src, src_level, sz, sfmt, eqfmt);
if (ret)
return ret;
IMMED_NVC0(push, NVC0_2D(BLIT_CONTROL), 0x00);
BEGIN_NVC0(push, NVC0_2D(BLIT_DST_X), 4);
PUSH_DATA (push, dx << dst->ms_x);
PUSH_DATA (push, dy << dst->ms_y);
PUSH_DATA (push, w << dst->ms_x);
PUSH_DATA (push, h << dst->ms_y);
BEGIN_NVC0(push, NVC0_2D(BLIT_DU_DX_FRACT), 4);
PUSH_DATA (push, 0);
PUSH_DATA (push, 1);
PUSH_DATA (push, 0);
PUSH_DATA (push, 1);
BEGIN_NVC0(push, NVC0_2D(BLIT_SRC_X_FRACT), 4);
PUSH_DATA (push, 0);
PUSH_DATA (push, sx << src->ms_x);
PUSH_DATA (push, 0);
PUSH_DATA (push, sy << src->ms_x);
return 0;
}
static inline void
nvc0_compute_invalidate_surfaces(struct nvc0_context *nvc0, const int s)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
int i;
for (i = 0; i < NVC0_MAX_IMAGES; ++i) {
if (s == 5)
BEGIN_NVC0(push, NVC0_CP(IMAGE(i)), 6);
else
BEGIN_NVC0(push, NVC0_3D(IMAGE(i)), 6);
PUSH_DATA(push, 0);
PUSH_DATA(push, 0);
PUSH_DATA(push, 0);
PUSH_DATA(push, 0);
PUSH_DATA(push, 0x14000);
PUSH_DATA(push, 0);
}
}
void
nvc0_vertprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *vp = nvc0->vertprog;
if (!nvc0_program_validate(nvc0, vp))
return;
nvc0_program_update_context_state(nvc0, vp, 0);
BEGIN_NVC0(push, NVC0_3D(SP_SELECT(1)), 2);
PUSH_DATA (push, 0x11);
PUSH_DATA (push, vp->code_base);
BEGIN_NVC0(push, NVC0_3D(SP_GPR_ALLOC(1)), 1);
PUSH_DATA (push, vp->max_gpr);
// BEGIN_NVC0(push, NVC0_3D_(0x163c), 1);
// PUSH_DATA (push, 0);
}
void
nvc0_compprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *cp = nvc0->compprog;
if (cp && !nvc0_program_validate(nvc0, cp))
return;
BEGIN_NVC0(push, NVC0_CP(FLUSH), 1);
PUSH_DATA (push, NVC0_COMPUTE_FLUSH_CODE);
}
static void
nvc0_memcpy_m2mf(struct pscnv_ib_chan *chan, const uint64_t dst_addr,
const uint64_t src_addr, const uint32_t size, int flags)
{
/* MODE 1 means fire fence */
static const uint32_t mode1 = 0x102110; /* QUERY_SHORT|QUERY_YES|SRC_LINEAR|DST_LINEAR */
static const uint32_t mode2 = 0x100110; /* QUERY_SHORT|SRC_LINEAR|DST_LINEAR */
static const uint32_t page_size = PSCNV_MEM_PAGE_SIZE;
const uint32_t page_count = size / page_size;
const uint32_t rem_size = size - page_size * page_count;
uint64_t dst_pos = dst_addr;
uint64_t src_pos = src_addr;
uint32_t pages_left = page_count;
if (flags & PSCNV_DMA_VERBOSE) {
char size_str[16];
pscnv_mem_human_readable(size_str, size);
NV_INFO(chan->dev, "DMA: M2MF- copy %s from %llx to %llx\n",
size_str, src_addr, dst_addr);
}
while (pages_left) {
int line_count = (pages_left > 2047) ? 2047 : pages_left;
BEGIN_NVC0(chan, GDEV_SUBCH_NV_M2MF, 0x238, 2);
OUT_RING(chan, dst_pos >> 32); /* OFFSET_OUT_HIGH */
OUT_RING(chan, dst_pos); /* OFFSET_OUT_LOW */
BEGIN_NVC0(chan, GDEV_SUBCH_NV_M2MF, 0x30c, 6);
OUT_RING(chan, src_pos >> 32); /* OFFSET_IN_HIGH */
OUT_RING(chan, src_pos); /* OFFSET_IN_LOW */
OUT_RING(chan, page_size); /* SRC_PITCH_IN */
OUT_RING(chan, page_size); /* DST_PITCH_IN */
OUT_RING(chan, page_size); /* LINE_LENGTH_IN */
OUT_RING(chan, line_count); /* LINE_COUNT */
BEGIN_NVC0(chan, GDEV_SUBCH_NV_M2MF, 0x300, 1);
if (pages_left == line_count && rem_size == 0)
OUT_RING(chan, mode1); /* EXEC */
else
OUT_RING(chan, mode2); /* EXEC */
pages_left -= line_count;
dst_pos += (page_size * line_count);
src_pos += (page_size * line_count);
}
if (rem_size) {
BEGIN_NVC0(chan, GDEV_SUBCH_NV_M2MF, 0x238, 2);
OUT_RING(chan, dst_pos >> 32); /* OFFSET_OUT_HIGH */
OUT_RING(chan, dst_pos); /* OFFSET_OUT_LOW */
BEGIN_NVC0(chan, GDEV_SUBCH_NV_M2MF, 0x30c, 6);
OUT_RING(chan, src_pos >> 32); /* OFFSET_IN_HIGH */
OUT_RING(chan, src_pos); /* OFFSET_IN_LOW */
OUT_RING(chan, rem_size); /* SRC_PITCH_IN */
OUT_RING(chan, rem_size); /* DST_PITCH_IN */
OUT_RING(chan, rem_size); /* LINE_LENGTH_IN */
OUT_RING(chan, 1); /* LINE_COUNT */
BEGIN_NVC0(chan, GDEV_SUBCH_NV_M2MF, 0x300, 1);
OUT_RING(chan, mode1); /* EXEC */
}
FIRE_RING(chan);
}
void
nve4_p2mf_push_linear(struct nouveau_context *nv,
struct nouveau_bo *dst, unsigned offset, unsigned domain,
unsigned size, const void *data)
{
struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
struct nouveau_pushbuf *push = nv->pushbuf;
uint32_t *src = (uint32_t *)data;
unsigned count = (size + 3) / 4;
nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
nouveau_pushbuf_bufctx(push, nvc0->bufctx);
nouveau_pushbuf_validate(push);
while (count) {
unsigned nr = MIN2(count, (NV04_PFIFO_MAX_PACKET_LEN - 1));
if (!PUSH_SPACE(push, nr + 10))
break;
BEGIN_NVC0(push, NVE4_P2MF(UPLOAD_DST_ADDRESS_HIGH), 2);
PUSH_DATAh(push, dst->offset + offset);
PUSH_DATA (push, dst->offset + offset);
BEGIN_NVC0(push, NVE4_P2MF(UPLOAD_LINE_LENGTH_IN), 2);
PUSH_DATA (push, MIN2(size, nr * 4));
PUSH_DATA (push, 1);
/* must not be interrupted (trap on QUERY fence, 0x50 works however) */
BEGIN_1IC0(push, NVE4_P2MF(UPLOAD_EXEC), nr + 1);
PUSH_DATA (push, 0x1001);
PUSH_DATAp(push, src, nr);
count -= nr;
src += nr;
offset += nr * 4;
size -= nr * 4;
}
nouveau_bufctx_reset(nvc0->bufctx, 0);
}
static void
nvc0_compute_validate_constbufs(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
const int s = 5;
while (nvc0->constbuf_dirty[s]) {
int i = ffs(nvc0->constbuf_dirty[s]) - 1;
nvc0->constbuf_dirty[s] &= ~(1 << i);
if (nvc0->constbuf[s][i].user) {
struct nouveau_bo *bo = nvc0->screen->uniform_bo;
const unsigned base = NVC0_CB_USR_INFO(s);
const unsigned size = nvc0->constbuf[s][0].size;
assert(i == 0); /* we really only want OpenGL uniforms here */
assert(nvc0->constbuf[s][0].u.data);
if (nvc0->state.uniform_buffer_bound[s] < size) {
nvc0->state.uniform_buffer_bound[s] = align(size, 0x100);
BEGIN_NVC0(push, NVC0_CP(CB_SIZE), 3);
PUSH_DATA (push, nvc0->state.uniform_buffer_bound[s]);
PUSH_DATAh(push, bo->offset + base);
PUSH_DATA (push, bo->offset + base);
BEGIN_NVC0(push, NVC0_CP(CB_BIND), 1);
PUSH_DATA (push, (0 << 8) | 1);
}
nvc0_cb_bo_push(&nvc0->base, bo, NV_VRAM_DOMAIN(&nvc0->screen->base),
base, nvc0->state.uniform_buffer_bound[s],
0, (size + 3) / 4,
nvc0->constbuf[s][0].u.data);
} else {
struct nv04_resource *res =
nv04_resource(nvc0->constbuf[s][i].u.buf);
if (res) {
BEGIN_NVC0(push, NVC0_CP(CB_SIZE), 3);
PUSH_DATA (push, nvc0->constbuf[s][i].size);
PUSH_DATAh(push, res->address + nvc0->constbuf[s][i].offset);
PUSH_DATA (push, res->address + nvc0->constbuf[s][i].offset);
BEGIN_NVC0(push, NVC0_CP(CB_BIND), 1);
PUSH_DATA (push, (i << 8) | 1);
BCTX_REFN(nvc0->bufctx_cp, CP_CB(i), res, RD);
res->cb_bindings[s] |= 1 << i;
} else {
BEGIN_NVC0(push, NVC0_CP(CB_BIND), 1);
PUSH_DATA (push, (i << 8) | 0);
}
if (i == 0)
nvc0->state.uniform_buffer_bound[s] = 0;
}
}
nvc0_compute_invalidate_constbufs(nvc0);
BEGIN_NVC0(push, NVC0_CP(FLUSH), 1);
PUSH_DATA (push, NVC0_COMPUTE_FLUSH_CB);
}
static void
nvc0_compute_upload_input(struct nvc0_context *nvc0,
const struct pipe_grid_info *info)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_screen *screen = nvc0->screen;
struct nvc0_program *cp = nvc0->compprog;
if (cp->parm_size) {
struct nouveau_bo *bo = screen->uniform_bo;
const unsigned base = NVC0_CB_USR_INFO(5);
BEGIN_NVC0(push, NVC0_CP(CB_SIZE), 3);
PUSH_DATA (push, align(cp->parm_size, 0x100));
PUSH_DATAh(push, bo->offset + base);
PUSH_DATA (push, bo->offset + base);
BEGIN_NVC0(push, NVC0_CP(CB_BIND), 1);
PUSH_DATA (push, (0 << 8) | 1);
/* NOTE: size is limited to 4 KiB, which is < NV04_PFIFO_MAX_PACKET_LEN */
BEGIN_1IC0(push, NVC0_CP(CB_POS), 1 + cp->parm_size / 4);
PUSH_DATA (push, 0);
PUSH_DATAp(push, info->input, cp->parm_size / 4);
nvc0_compute_invalidate_constbufs(nvc0);
}
BEGIN_NVC0(push, NVC0_CP(CB_SIZE), 3);
PUSH_DATA (push, NVC0_CB_AUX_SIZE);
PUSH_DATAh(push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(5));
PUSH_DATA (push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(5));
BEGIN_1IC0(push, NVC0_CP(CB_POS), 1 + 1);
/* (7) as we only upload work_dim on nvc0, the rest uses special regs */
PUSH_DATA (push, NVC0_CB_AUX_GRID_INFO(7));
PUSH_DATA (push, info->work_dim);
BEGIN_NVC0(push, NVC0_CP(FLUSH), 1);
PUSH_DATA (push, NVC0_COMPUTE_FLUSH_CB);
}
void
nvc0_tevlprog_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *tp = nvc0->tevlprog;
if (tp && nvc0_program_validate(nvc0, tp)) {
if (tp->tp.tess_mode != ~0) {
BEGIN_NVC0(push, NVC0_3D(TESS_MODE), 1);
PUSH_DATA (push, tp->tp.tess_mode);
}
BEGIN_NVC0(push, NVC0_3D(MACRO_TEP_SELECT), 1);
PUSH_DATA (push, 0x31);
BEGIN_NVC0(push, NVC0_3D(SP_START_ID(3)), 1);
PUSH_DATA (push, tp->code_base);
BEGIN_NVC0(push, NVC0_3D(SP_GPR_ALLOC(3)), 1);
PUSH_DATA (push, tp->max_gpr);
} else {
BEGIN_NVC0(push, NVC0_3D(MACRO_TEP_SELECT), 1);
PUSH_DATA (push, 0x30);
}
nvc0_program_update_context_state(nvc0, tp, 2);
}