void
nve4_p2mf_push_linear(struct nouveau_context *nv,
struct nouveau_bo *dst, unsigned offset, unsigned domain,
unsigned size, const void *data)
{
struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
struct nouveau_pushbuf *push = nv->pushbuf;
uint32_t *src = (uint32_t *)data;
unsigned count = (size + 3) / 4;
nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
nouveau_pushbuf_bufctx(push, nvc0->bufctx);
nouveau_pushbuf_validate(push);
while (count) {
unsigned nr = MIN2(count, (NV04_PFIFO_MAX_PACKET_LEN - 1));
if (!PUSH_SPACE(push, nr + 10))
break;
BEGIN_NVC0(push, NVE4_P2MF(UPLOAD_DST_ADDRESS_HIGH), 2);
PUSH_DATAh(push, dst->offset + offset);
PUSH_DATA (push, dst->offset + offset);
BEGIN_NVC0(push, NVE4_P2MF(UPLOAD_LINE_LENGTH_IN), 2);
PUSH_DATA (push, MIN2(size, nr * 4));
PUSH_DATA (push, 1);
/* must not be interrupted (trap on QUERY fence, 0x50 works however) */
BEGIN_1IC0(push, NVE4_P2MF(UPLOAD_EXEC), nr + 1);
PUSH_DATA (push, 0x1001);
PUSH_DATAp(push, src, nr);
count -= nr;
src += nr;
offset += nr * 4;
size -= nr * 4;
}
nouveau_bufctx_reset(nvc0->bufctx, 0);
}
static void
nv50_set_index_buffer(struct pipe_context *pipe,
const struct pipe_index_buffer *ib)
{
struct nv50_context *nv50 = nv50_context(pipe);
if (nv50->idxbuf.buffer)
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_INDEX);
if (ib) {
pipe_resource_reference(&nv50->idxbuf.buffer, ib->buffer);
nv50->idxbuf.index_size = ib->index_size;
if (ib->buffer) {
nv50->idxbuf.offset = ib->offset;
BCTX_REFN(nv50->bufctx_3d, INDEX, nv04_resource(ib->buffer), RD);
} else {
nv50->idxbuf.user_buffer = ib->user_buffer;
}
} else {
pipe_resource_reference(&nv50->idxbuf.buffer, NULL);
}
}
static void
nv50_set_constant_buffer(struct pipe_context *pipe, uint shader, uint index,
struct pipe_constant_buffer *cb)
{
struct nv50_context *nv50 = nv50_context(pipe);
struct pipe_resource *res = cb ? cb->buffer : NULL;
const unsigned s = nv50_context_shader_stage(shader);
const unsigned i = index;
if (shader == PIPE_SHADER_COMPUTE)
return;
assert(i < NV50_MAX_PIPE_CONSTBUFS);
if (nv50->constbuf[s][i].user)
nv50->constbuf[s][i].u.buf = NULL;
else
if (nv50->constbuf[s][i].u.buf)
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_CB(s, i));
pipe_resource_reference(&nv50->constbuf[s][i].u.buf, res);
nv50->constbuf[s][i].user = (cb && cb->user_buffer) ? TRUE : FALSE;
if (nv50->constbuf[s][i].user) {
nv50->constbuf[s][i].u.data = cb->user_buffer;
nv50->constbuf[s][i].size = MIN2(cb->buffer_size, 0x10000);
nv50->constbuf_valid[s] |= 1 << i;
} else
if (res) {
nv50->constbuf[s][i].offset = cb->buffer_offset;
nv50->constbuf[s][i].size = MIN2(align(cb->buffer_size, 0x100), 0x10000);
nv50->constbuf_valid[s] |= 1 << i;
} else {
nv50->constbuf_valid[s] &= ~(1 << i);
}
nv50->constbuf_dirty[s] |= 1 << i;
nv50->dirty |= NV50_NEW_CONSTBUF;
}
static void
nv50_set_framebuffer_state(struct pipe_context *pipe,
const struct pipe_framebuffer_state *fb)
{
struct nv50_context *nv50 = nv50_context(pipe);
unsigned i;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_FB);
for (i = 0; i < fb->nr_cbufs; ++i)
pipe_surface_reference(&nv50->framebuffer.cbufs[i], fb->cbufs[i]);
for (; i < nv50->framebuffer.nr_cbufs; ++i)
pipe_surface_reference(&nv50->framebuffer.cbufs[i], NULL);
nv50->framebuffer.nr_cbufs = fb->nr_cbufs;
nv50->framebuffer.width = fb->width;
nv50->framebuffer.height = fb->height;
pipe_surface_reference(&nv50->framebuffer.zsbuf, fb->zsbuf);
nv50->dirty |= NV50_NEW_FRAMEBUFFER;
}
static void
nvc0_m2mf_copy_linear(struct nouveau_context *nv,
struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
unsigned size)
{
struct nouveau_pushbuf *push = nv->pushbuf;
struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
nouveau_pushbuf_bufctx(push, bctx);
nouveau_pushbuf_validate(push);
while (size) {
unsigned bytes = MIN2(size, 1 << 17);
BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
PUSH_DATAh(push, dst->offset + dstoff);
PUSH_DATA (push, dst->offset + dstoff);
BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
PUSH_DATAh(push, src->offset + srcoff);
PUSH_DATA (push, src->offset + srcoff);
BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
PUSH_DATA (push, bytes);
PUSH_DATA (push, 1);
BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
srcoff += bytes;
dstoff += bytes;
size -= bytes;
}
nouveau_bufctx_reset(bctx, 0);
}
void
nvc0_cb_push(struct nouveau_context *nv,
struct nouveau_bo *bo, unsigned domain,
unsigned base, unsigned size,
unsigned offset, unsigned words, const uint32_t *data)
{
struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
struct nouveau_pushbuf *push = nv->pushbuf;
assert(!(offset & 3));
size = align(size, 0x100);
nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
nouveau_pushbuf_bufctx(push, bctx);
nouveau_pushbuf_validate(push);
BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
PUSH_DATA (push, size);
PUSH_DATAh(push, bo->offset + base);
PUSH_DATA (push, bo->offset + base);
while (words) {
unsigned nr = PUSH_AVAIL(push);
nr = MIN2(nr, words);
nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
PUSH_DATA (push, offset);
PUSH_DATAp(push, data, nr);
words -= nr;
data += nr;
offset += nr * 4;
}
nouveau_bufctx_reset(bctx, 0);
}
static void
nv50_set_vertex_buffers(struct pipe_context *pipe,
unsigned start_slot, unsigned count,
const struct pipe_vertex_buffer *vb)
{
struct nv50_context *nv50 = nv50_context(pipe);
unsigned i;
util_set_vertex_buffers_count(nv50->vtxbuf, &nv50->num_vtxbufs, vb,
start_slot, count);
if (!vb) {
nv50->vbo_user &= ~(((1ull << count) - 1) << start_slot);
nv50->vbo_constant &= ~(((1ull << count) - 1) << start_slot);
return;
}
for (i = 0; i < count; ++i) {
unsigned dst_index = start_slot + i;
if (!vb[i].buffer && vb[i].user_buffer) {
nv50->vbo_user |= 1 << dst_index;
if (!vb[i].stride)
nv50->vbo_constant |= 1 << dst_index;
else
nv50->vbo_constant &= ~(1 << dst_index);
} else {
nv50->vbo_user &= ~(1 << dst_index);
nv50->vbo_constant &= ~(1 << dst_index);
}
}
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_VERTEX);
nv50->dirty |= NV50_NEW_ARRAYS;
}
static void
nvc0_set_index_buffer(struct pipe_context *pipe,
const struct pipe_index_buffer *ib)
{
struct nvc0_context *nvc0 = nvc0_context(pipe);
if (nvc0->idxbuf.buffer)
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_IDX);
if (ib) {
pipe_resource_reference(&nvc0->idxbuf.buffer, ib->buffer);
nvc0->idxbuf.index_size = ib->index_size;
if (ib->buffer) {
nvc0->idxbuf.offset = ib->offset;
nvc0->dirty_3d |= NVC0_NEW_3D_IDXBUF;
} else {
nvc0->idxbuf.user_buffer = ib->user_buffer;
nvc0->dirty_3d &= ~NVC0_NEW_3D_IDXBUF;
}
} else {
nvc0->dirty_3d &= ~NVC0_NEW_3D_IDXBUF;
pipe_resource_reference(&nvc0->idxbuf.buffer, NULL);
}
}
void
nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
{
struct push_context ctx;
unsigned i, index_size;
unsigned inst_count = info->instance_count;
unsigned vert_count = info->count;
unsigned prim;
nvc0_push_context_init(nvc0, &ctx);
nvc0_vertex_configure_translate(nvc0, info->index_bias);
if (nvc0->state.index_bias) {
/* this is already taken care of by translate */
IMMED_NVC0(ctx.push, NVC0_3D(VB_ELEMENT_BASE), 0);
nvc0->state.index_bias = 0;
}
if (unlikely(ctx.edgeflag.enabled))
nvc0_push_map_edgeflag(&ctx, nvc0, info->index_bias);
ctx.prim_restart = info->primitive_restart;
ctx.restart_index = info->restart_index;
if (info->primitive_restart) {
/* NOTE: I hope we won't ever need that last index (~0).
* If we do, we have to disable primitive restart here always and
* use END,BEGIN to restart. (XXX: would that affect PrimitiveID ?)
* We could also deactive PRIM_RESTART_WITH_DRAW_ARRAYS temporarily,
* and add manual restart to disp_vertices_seq.
*/
BEGIN_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
PUSH_DATA (ctx.push, 1);
PUSH_DATA (ctx.push, info->indexed ? 0xffffffff : info->restart_index);
} else
if (nvc0->state.prim_restart) {
IMMED_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
}
nvc0->state.prim_restart = info->primitive_restart;
if (info->indexed) {
nvc0_push_map_idxbuf(&ctx, nvc0);
index_size = nvc0->idxbuf.index_size;
} else {
if (unlikely(info->count_from_stream_output)) {
struct pipe_context *pipe = &nvc0->base.pipe;
struct nvc0_so_target *targ;
targ = nvc0_so_target(info->count_from_stream_output);
pipe->get_query_result(pipe, targ->pq, true, (void *)&vert_count);
vert_count /= targ->stride;
}
ctx.idxbuf = NULL; /* shut up warnings */
index_size = 0;
}
ctx.instance_id = info->start_instance;
prim = nvc0_prim_gl(info->mode);
do {
PUSH_SPACE(ctx.push, 9);
ctx.dest = nvc0_push_setup_vertex_array(nvc0, vert_count);
if (unlikely(!ctx.dest))
break;
if (unlikely(ctx.need_vertex_id))
nvc0_push_upload_vertex_ids(&ctx, nvc0, info);
if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_BEGIN_GL), 1);
PUSH_DATA (ctx.push, prim);
switch (index_size) {
case 1:
disp_vertices_i08(&ctx, info->start, vert_count);
break;
case 2:
disp_vertices_i16(&ctx, info->start, vert_count);
break;
case 4:
disp_vertices_i32(&ctx, info->start, vert_count);
break;
default:
assert(index_size == 0);
disp_vertices_seq(&ctx, info->start, vert_count);
break;
}
PUSH_SPACE(ctx.push, 1);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_END_GL), 0);
if (--inst_count) {
prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
++ctx.instance_id;
}
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX_TMP);
nouveau_scratch_done(&nvc0->base);
} while (inst_count);
/* reset state and unmap buffers (no-op) */
if (unlikely(!ctx.edgeflag.value)) {
PUSH_SPACE(ctx.push, 1);
IMMED_NVC0(ctx.push, NVC0_3D(EDGEFLAG), 1);
}
if (unlikely(ctx.need_vertex_id)) {
PUSH_SPACE(ctx.push, 4);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ID_REPLACE), 0);
BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_ATTRIB_FORMAT(1)), 1);
PUSH_DATA (ctx.push,
NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST |
NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT |
NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 0);
}
if (info->indexed)
nouveau_resource_unmap(nv04_resource(nvc0->idxbuf.buffer));
for (i = 0; i < nvc0->num_vtxbufs; ++i)
nouveau_resource_unmap(nv04_resource(nvc0->vtxbuf[i].buffer));
NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_fallback_count, 1);
}
void
nvc0_tfb_validate(struct nvc0_context *nvc0)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_transform_feedback_state *tfb;
unsigned b;
if (nvc0->gmtyprog) tfb = nvc0->gmtyprog->tfb;
else
if (nvc0->tevlprog) tfb = nvc0->tevlprog->tfb;
else
tfb = nvc0->vertprog->tfb;
IMMED_NVC0(push, NVC0_3D(TFB_ENABLE), (tfb && nvc0->num_tfbbufs) ? 1 : 0);
if (tfb && tfb != nvc0->state.tfb) {
for (b = 0; b < 4; ++b) {
if (tfb->varying_count[b]) {
unsigned n = (tfb->varying_count[b] + 3) / 4;
BEGIN_NVC0(push, NVC0_3D(TFB_STREAM(b)), 3);
PUSH_DATA (push, 0);
PUSH_DATA (push, tfb->varying_count[b]);
PUSH_DATA (push, tfb->stride[b]);
BEGIN_NVC0(push, NVC0_3D(TFB_VARYING_LOCS(b, 0)), n);
PUSH_DATAp(push, tfb->varying_index[b], n);
if (nvc0->tfbbuf[b])
nvc0_so_target(nvc0->tfbbuf[b])->stride = tfb->stride[b];
} else {
IMMED_NVC0(push, NVC0_3D(TFB_VARYING_COUNT(b)), 0);
}
}
}
nvc0->state.tfb = tfb;
if (!(nvc0->dirty & NVC0_NEW_TFB_TARGETS))
return;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_TFB);
for (b = 0; b < nvc0->num_tfbbufs; ++b) {
struct nvc0_so_target *targ = nvc0_so_target(nvc0->tfbbuf[b]);
struct nv04_resource *buf = nv04_resource(targ->pipe.buffer);
if (tfb)
targ->stride = tfb->stride[b];
if (!(nvc0->tfbbuf_dirty & (1 << b)))
continue;
if (!targ->clean)
nvc0_query_fifo_wait(push, targ->pq);
BEGIN_NVC0(push, NVC0_3D(TFB_BUFFER_ENABLE(b)), 5);
PUSH_DATA (push, 1);
PUSH_DATAh(push, buf->address + targ->pipe.buffer_offset);
PUSH_DATA (push, buf->address + targ->pipe.buffer_offset);
PUSH_DATA (push, targ->pipe.buffer_size);
if (!targ->clean) {
nvc0_query_pushbuf_submit(push, targ->pq, 0x4);
} else {
PUSH_DATA(push, 0); /* TFB_BUFFER_OFFSET */
targ->clean = FALSE;
}
BCTX_REFN(nvc0->bufctx_3d, TFB, buf, WR);
}
for (; b < 4; ++b)
IMMED_NVC0(push, NVC0_3D(TFB_BUFFER_ENABLE(b)), 0);
}
static void
nvc0_resource_copy_region(struct pipe_context *pipe,
struct pipe_resource *dst, unsigned dst_level,
unsigned dstx, unsigned dsty, unsigned dstz,
struct pipe_resource *src, unsigned src_level,
const struct pipe_box *src_box)
{
struct nvc0_context *nvc0 = nvc0_context(pipe);
int ret;
boolean m2mf;
unsigned dst_layer = dstz, src_layer = src_box->z;
if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
nouveau_copy_buffer(&nvc0->base,
nv04_resource(dst), dstx,
nv04_resource(src), src_box->x, src_box->width);
NOUVEAU_DRV_STAT(&nvc0->screen->base, buf_copy_bytes, src_box->width);
return;
}
NOUVEAU_DRV_STAT(&nvc0->screen->base, tex_copy_count, 1);
/* 0 and 1 are equal, only supporting 0/1, 2, 4 and 8 */
assert((src->nr_samples | 1) == (dst->nr_samples | 1));
m2mf = (src->format == dst->format) ||
(util_format_get_blocksizebits(src->format) ==
util_format_get_blocksizebits(dst->format));
nv04_resource(dst)->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
if (m2mf) {
struct nv50_m2mf_rect drect, srect;
unsigned i;
unsigned nx = util_format_get_nblocksx(src->format, src_box->width);
unsigned ny = util_format_get_nblocksy(src->format, src_box->height);
nv50_m2mf_rect_setup(&drect, dst, dst_level, dstx, dsty, dstz);
nv50_m2mf_rect_setup(&srect, src, src_level,
src_box->x, src_box->y, src_box->z);
for (i = 0; i < src_box->depth; ++i) {
nvc0->m2mf_copy_rect(nvc0, &drect, &srect, nx, ny);
if (nv50_miptree(dst)->layout_3d)
drect.z++;
else
drect.base += nv50_miptree(dst)->layer_stride;
if (nv50_miptree(src)->layout_3d)
srect.z++;
else
srect.base += nv50_miptree(src)->layer_stride;
}
return;
}
assert(nv50_2d_dst_format_faithful(dst->format));
assert(nv50_2d_src_format_faithful(src->format));
BCTX_REFN(nvc0->bufctx, 2D, nv04_resource(src), RD);
BCTX_REFN(nvc0->bufctx, 2D, nv04_resource(dst), WR);
nouveau_pushbuf_bufctx(nvc0->base.pushbuf, nvc0->bufctx);
nouveau_pushbuf_validate(nvc0->base.pushbuf);
for (; dst_layer < dstz + src_box->depth; ++dst_layer, ++src_layer) {
ret = nvc0_2d_texture_do_copy(nvc0->base.pushbuf,
nv50_miptree(dst), dst_level,
dstx, dsty, dst_layer,
nv50_miptree(src), src_level,
src_box->x, src_box->y, src_layer,
src_box->width, src_box->height);
if (ret)
break;
}
nouveau_bufctx_reset(nvc0->bufctx, 0);
}
static bool
nvc0_bind_images_range(struct nvc0_context *nvc0, const unsigned s,
unsigned start, unsigned nr,
struct pipe_image_view *pimages)
{
const unsigned end = start + nr;
unsigned mask = 0;
unsigned i;
assert(s < 6);
if (pimages) {
for (i = start; i < end; ++i) {
struct pipe_image_view *img = &nvc0->images[s][i];
const unsigned p = i - start;
if (img->resource == pimages[p].resource &&
img->format == pimages[p].format &&
img->access == pimages[p].access) {
if (img->resource == NULL)
continue;
if (img->resource->target == PIPE_BUFFER &&
img->u.buf.first_element == pimages[p].u.buf.first_element &&
img->u.buf.last_element == pimages[p].u.buf.last_element)
continue;
if (img->resource->target != PIPE_BUFFER &&
img->u.tex.first_layer == pimages[p].u.tex.first_layer &&
img->u.tex.last_layer == pimages[p].u.tex.last_layer &&
img->u.tex.level == pimages[p].u.tex.level)
continue;
}
mask |= (1 << i);
if (pimages[p].resource)
nvc0->images_valid[s] |= (1 << i);
else
nvc0->images_valid[s] &= ~(1 << i);
img->format = pimages[p].format;
img->access = pimages[p].access;
if (pimages[p].resource && pimages[p].resource->target == PIPE_BUFFER)
img->u.buf = pimages[p].u.buf;
else
img->u.tex = pimages[p].u.tex;
pipe_resource_reference(
&img->resource, pimages[p].resource);
}
if (!mask)
return false;
} else {
mask = ((1 << nr) - 1) << start;
if (!(nvc0->images_valid[s] & mask))
return false;
for (i = start; i < end; ++i)
pipe_resource_reference(&nvc0->images[s][i].resource, NULL);
nvc0->images_valid[s] &= ~mask;
}
nvc0->images_dirty[s] |= mask;
if (s == 5)
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_SUF);
else
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_SUF);
return true;
}
static int
nv50_invalidate_resource_storage(struct nouveau_context *ctx,
struct pipe_resource *res,
int ref)
{
struct nv50_context *nv50 = nv50_context(&ctx->pipe);
unsigned s, i;
if (res->bind & PIPE_BIND_RENDER_TARGET) {
for (i = 0; i < nv50->framebuffer.nr_cbufs; ++i) {
if (nv50->framebuffer.cbufs[i] &&
nv50->framebuffer.cbufs[i]->texture == res) {
nv50->dirty |= NV50_NEW_FRAMEBUFFER;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_FB);
if (!--ref)
return ref;
}
}
}
if (res->bind & PIPE_BIND_DEPTH_STENCIL) {
if (nv50->framebuffer.zsbuf &&
nv50->framebuffer.zsbuf->texture == res) {
nv50->dirty |= NV50_NEW_FRAMEBUFFER;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_FB);
if (!--ref)
return ref;
}
}
if (res->bind & PIPE_BIND_VERTEX_BUFFER) {
for (i = 0; i < nv50->num_vtxbufs; ++i) {
if (nv50->vtxbuf[i].buffer == res) {
nv50->dirty |= NV50_NEW_ARRAYS;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_VERTEX);
if (!--ref)
return ref;
}
}
}
if (res->bind & PIPE_BIND_INDEX_BUFFER) {
if (nv50->idxbuf.buffer == res)
if (!--ref)
return ref;
}
if (res->bind & PIPE_BIND_SAMPLER_VIEW) {
for (s = 0; s < 5; ++s) {
for (i = 0; i < nv50->num_textures[s]; ++i) {
if (nv50->textures[s][i] &&
nv50->textures[s][i]->texture == res) {
nv50->dirty |= NV50_NEW_TEXTURES;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_TEXTURES);
if (!--ref)
return ref;
}
}
}
}
if (res->bind & PIPE_BIND_CONSTANT_BUFFER) {
for (s = 0; s < 5; ++s) {
for (i = 0; i < nv50->num_vtxbufs; ++i) {
if (!nv50->constbuf[s][i].user &&
nv50->constbuf[s][i].u.buf == res) {
nv50->dirty |= NV50_NEW_CONSTBUF;
nv50->constbuf_dirty[s] |= 1 << i;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_CB(s, i));
if (!--ref)
return ref;
}
}
}
}
return ref;
}
static void
nv50_hw_sm_end_query(struct nv50_context *nv50, struct nv50_hw_query *hq)
{
struct nv50_screen *screen = nv50->screen;
struct pipe_context *pipe = &nv50->base.pipe;
struct nouveau_pushbuf *push = nv50->base.pushbuf;
struct nv50_hw_sm_query *hsq = nv50_hw_sm_query(hq);
struct pipe_grid_info info = {};
uint32_t mask;
uint32_t input[3];
const uint block[3] = { 32, 1, 1 };
const uint grid[3] = { screen->MPsInTP, screen->TPs, 1 };
int c, i;
if (unlikely(!screen->pm.prog)) {
struct nv50_program *prog = CALLOC_STRUCT(nv50_program);
prog->type = PIPE_SHADER_COMPUTE;
prog->translated = true;
prog->max_gpr = 7;
prog->parm_size = 8;
prog->code = (uint32_t *)nv50_read_hw_sm_counters_code;
prog->code_size = sizeof(nv50_read_hw_sm_counters_code);
screen->pm.prog = prog;
}
/* disable all counting */
PUSH_SPACE(push, 8);
for (c = 0; c < 4; c++) {
if (screen->pm.mp_counter[c]) {
BEGIN_NV04(push, NV50_COMPUTE(MP_PM_CONTROL(c)), 1);
PUSH_DATA (push, 0);
}
}
/* release counters for this query */
for (c = 0; c < 4; c++) {
if (screen->pm.mp_counter[c] == hsq) {
screen->pm.num_hw_sm_active--;
screen->pm.mp_counter[c] = NULL;
}
}
BCTX_REFN_bo(nv50->bufctx_cp, CP_QUERY, NOUVEAU_BO_GART | NOUVEAU_BO_WR,
hq->bo);
PUSH_SPACE(push, 2);
BEGIN_NV04(push, SUBC_COMPUTE(NV50_GRAPH_SERIALIZE), 1);
PUSH_DATA (push, 0);
pipe->bind_compute_state(pipe, screen->pm.prog);
input[0] = hq->bo->offset + hq->base_offset;
input[1] = hq->sequence;
for (i = 0; i < 3; i++) {
info.block[i] = block[i];
info.grid[i] = grid[i];
}
info.pc = 0;
info.input = input;
pipe->launch_grid(pipe, &info);
nouveau_bufctx_reset(nv50->bufctx_cp, NV50_BIND_CP_QUERY);
/* re-active other counters */
PUSH_SPACE(push, 8);
mask = 0;
for (c = 0; c < 4; c++) {
const struct nv50_hw_sm_query_cfg *cfg;
unsigned i;
hsq = screen->pm.mp_counter[c];
if (!hsq)
continue;
cfg = nv50_hw_sm_query_get_cfg(nv50, &hsq->base);
for (i = 0; i < cfg->num_counters; i++) {
uint16_t func;
if (mask & (1 << hsq->ctr[i]))
break;
mask |= 1 << hsq->ctr[i];
func = nv50_hw_sm_get_func(hsq->ctr[i]);
BEGIN_NV04(push, NV50_COMPUTE(MP_PM_CONTROL(hsq->ctr[i])), 1);
PUSH_DATA (push, (cfg->ctr[i].sig << 24) | (func << 8)
| cfg->ctr[i].unit | cfg->ctr[i].mode);
}
}
}
static bool
nvc0_bind_images_range(struct nvc0_context *nvc0, const unsigned s,
unsigned start, unsigned nr,
const struct pipe_image_view *pimages)
{
const unsigned end = start + nr;
unsigned mask = 0;
unsigned i;
assert(s < 6);
if (pimages) {
for (i = start; i < end; ++i) {
struct pipe_image_view *img = &nvc0->images[s][i];
const unsigned p = i - start;
if (img->resource == pimages[p].resource &&
img->format == pimages[p].format &&
img->access == pimages[p].access) {
if (img->resource == NULL)
continue;
if (img->resource->target == PIPE_BUFFER &&
img->u.buf.offset == pimages[p].u.buf.offset &&
img->u.buf.size == pimages[p].u.buf.size)
continue;
if (img->resource->target != PIPE_BUFFER &&
img->u.tex.first_layer == pimages[p].u.tex.first_layer &&
img->u.tex.last_layer == pimages[p].u.tex.last_layer &&
img->u.tex.level == pimages[p].u.tex.level)
continue;
}
mask |= (1 << i);
if (pimages[p].resource)
nvc0->images_valid[s] |= (1 << i);
else
nvc0->images_valid[s] &= ~(1 << i);
img->format = pimages[p].format;
img->access = pimages[p].access;
if (pimages[p].resource && pimages[p].resource->target == PIPE_BUFFER)
img->u.buf = pimages[p].u.buf;
else
img->u.tex = pimages[p].u.tex;
pipe_resource_reference(
&img->resource, pimages[p].resource);
if (nvc0->screen->base.class_3d >= GM107_3D_CLASS) {
if (nvc0->images_tic[s][i]) {
struct nv50_tic_entry *old =
nv50_tic_entry(nvc0->images_tic[s][i]);
nvc0_screen_tic_unlock(nvc0->screen, old);
pipe_sampler_view_reference(&nvc0->images_tic[s][i], NULL);
}
nvc0->images_tic[s][i] =
gm107_create_texture_view_from_image(&nvc0->base.pipe,
&pimages[p]);
}
}
if (!mask)
return false;
} else {
mask = ((1 << nr) - 1) << start;
if (!(nvc0->images_valid[s] & mask))
return false;
for (i = start; i < end; ++i) {
pipe_resource_reference(&nvc0->images[s][i].resource, NULL);
if (nvc0->screen->base.class_3d >= GM107_3D_CLASS) {
struct nv50_tic_entry *old = nv50_tic_entry(nvc0->images_tic[s][i]);
if (old) {
nvc0_screen_tic_unlock(nvc0->screen, old);
pipe_sampler_view_reference(&nvc0->images_tic[s][i], NULL);
}
}
}
nvc0->images_valid[s] &= ~mask;
}
nvc0->images_dirty[s] |= mask;
if (s == 5)
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_SUF);
else
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_SUF);
return true;
}
void
nv30_vbo_validate(struct nv30_context *nv30)
{
struct nouveau_pushbuf *push = nv30->base.pushbuf;
struct nv30_vertex_stateobj *vertex = nv30->vertex;
struct pipe_vertex_element *ve;
struct pipe_vertex_buffer *vb;
unsigned i, redefine;
nouveau_bufctx_reset(nv30->bufctx, BUFCTX_VTXBUF);
if (!nv30->vertex || nv30->draw_flags)
return;
if (unlikely(vertex->need_conversion)) {
nv30->vbo_fifo = ~0;
nv30->vbo_user = 0;
} else {
nv30_prevalidate_vbufs(nv30);
}
if (!PUSH_SPACE(push, 128))
return;
redefine = MAX2(vertex->num_elements, nv30->state.num_vtxelts);
BEGIN_NV04(push, NV30_3D(VTXFMT(0)), redefine);
for (i = 0; i < vertex->num_elements; i++) {
ve = &vertex->pipe[i];
vb = &nv30->vtxbuf[ve->vertex_buffer_index];
if (likely(vb->stride) || nv30->vbo_fifo)
PUSH_DATA (push, (vb->stride << 8) | vertex->element[i].state);
else
PUSH_DATA (push, NV30_3D_VTXFMT_TYPE_V32_FLOAT);
}
for (; i < nv30->state.num_vtxelts; i++) {
PUSH_DATA (push, NV30_3D_VTXFMT_TYPE_V32_FLOAT);
}
for (i = 0; i < vertex->num_elements; i++) {
struct nv04_resource *res;
unsigned offset;
boolean user;
ve = &vertex->pipe[i];
vb = &nv30->vtxbuf[ve->vertex_buffer_index];
user = (nv30->vbo_user & (1 << ve->vertex_buffer_index));
res = nv04_resource(vb->buffer);
if (nv30->vbo_fifo || unlikely(vb->stride == 0)) {
if (!nv30->vbo_fifo)
nv30_emit_vtxattr(nv30, vb, ve, i);
continue;
}
offset = ve->src_offset + vb->buffer_offset;
BEGIN_NV04(push, NV30_3D(VTXBUF(i)), 1);
PUSH_RESRC(push, NV30_3D(VTXBUF(i)), user ? BUFCTX_VTXTMP : BUFCTX_VTXBUF,
res, offset, NOUVEAU_BO_LOW | NOUVEAU_BO_RD,
0, NV30_3D_VTXBUF_DMA1);
}
nv30->state.num_vtxelts = vertex->num_elements;
}
static int
nvc0_invalidate_resource_storage(struct nouveau_context *ctx,
struct pipe_resource *res,
int ref)
{
struct nvc0_context *nvc0 = nvc0_context(&ctx->pipe);
unsigned bind = res->bind ? res->bind : PIPE_BIND_VERTEX_BUFFER;
unsigned s, i;
if (bind & PIPE_BIND_RENDER_TARGET) {
for (i = 0; i < nvc0->framebuffer.nr_cbufs; ++i) {
if (nvc0->framebuffer.cbufs[i] &&
nvc0->framebuffer.cbufs[i]->texture == res) {
nvc0->dirty |= NVC0_NEW_FRAMEBUFFER;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_FB);
if (!--ref)
return ref;
}
}
}
if (bind & PIPE_BIND_DEPTH_STENCIL) {
if (nvc0->framebuffer.zsbuf &&
nvc0->framebuffer.zsbuf->texture == res) {
nvc0->dirty |= NVC0_NEW_FRAMEBUFFER;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_FB);
if (!--ref)
return ref;
}
}
if (bind & (PIPE_BIND_VERTEX_BUFFER |
PIPE_BIND_INDEX_BUFFER |
PIPE_BIND_CONSTANT_BUFFER |
PIPE_BIND_STREAM_OUTPUT |
PIPE_BIND_COMMAND_ARGS_BUFFER |
PIPE_BIND_SAMPLER_VIEW)) {
for (i = 0; i < nvc0->num_vtxbufs; ++i) {
if (nvc0->vtxbuf[i].buffer == res) {
nvc0->dirty |= NVC0_NEW_ARRAYS;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX);
if (!--ref)
return ref;
}
}
if (nvc0->idxbuf.buffer == res) {
nvc0->dirty |= NVC0_NEW_IDXBUF;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_IDX);
if (!--ref)
return ref;
}
for (s = 0; s < 5; ++s) {
for (i = 0; i < nvc0->num_textures[s]; ++i) {
if (nvc0->textures[s][i] &&
nvc0->textures[s][i]->texture == res) {
nvc0->textures_dirty[s] |= 1 << i;
nvc0->dirty |= NVC0_NEW_TEXTURES;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_TEX(s, i));
if (!--ref)
return ref;
}
}
}
for (s = 0; s < 5; ++s) {
for (i = 0; i < NVC0_MAX_PIPE_CONSTBUFS; ++i) {
if (!(nvc0->constbuf_valid[s] & (1 << i)))
continue;
if (!nvc0->constbuf[s][i].user &&
nvc0->constbuf[s][i].u.buf == res) {
nvc0->dirty |= NVC0_NEW_CONSTBUF;
nvc0->constbuf_dirty[s] |= 1 << i;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_CB(s, i));
if (!--ref)
return ref;
}
}
}
}
return ref;
}
static void
nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
const struct nv50_m2mf_rect *dst,
const struct nv50_m2mf_rect *src,
uint32_t nblocksx, uint32_t nblocksy)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nouveau_bufctx *bctx = nvc0->bufctx;
const int cpp = dst->cpp;
uint32_t src_ofst = src->base;
uint32_t dst_ofst = dst->base;
uint32_t height = nblocksy;
uint32_t sy = src->y;
uint32_t dy = dst->y;
uint32_t exec = (1 << 20);
assert(dst->cpp == src->cpp);
nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
nouveau_pushbuf_bufctx(push, bctx);
nouveau_pushbuf_validate(push);
if (nouveau_bo_memtype(src->bo)) {
BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
PUSH_DATA (push, src->tile_mode);
PUSH_DATA (push, src->width * cpp);
PUSH_DATA (push, src->height);
PUSH_DATA (push, src->depth);
PUSH_DATA (push, src->z);
} else {
src_ofst += src->y * src->pitch + src->x * cpp;
BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
PUSH_DATA (push, src->width * cpp);
exec |= NVC0_M2MF_EXEC_LINEAR_IN;
}
if (nouveau_bo_memtype(dst->bo)) {
BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
PUSH_DATA (push, dst->tile_mode);
PUSH_DATA (push, dst->width * cpp);
PUSH_DATA (push, dst->height);
PUSH_DATA (push, dst->depth);
PUSH_DATA (push, dst->z);
} else {
dst_ofst += dst->y * dst->pitch + dst->x * cpp;
BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
PUSH_DATA (push, dst->width * cpp);
exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
}
while (height) {
int line_count = height > 2047 ? 2047 : height;
BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
PUSH_DATAh(push, src->bo->offset + src_ofst);
PUSH_DATA (push, src->bo->offset + src_ofst);
BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
PUSH_DATAh(push, dst->bo->offset + dst_ofst);
PUSH_DATA (push, dst->bo->offset + dst_ofst);
if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
PUSH_DATA (push, src->x * cpp);
PUSH_DATA (push, sy);
} else {
src_ofst += line_count * src->pitch;
}
if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
PUSH_DATA (push, dst->x * cpp);
PUSH_DATA (push, dy);
} else {
dst_ofst += line_count * dst->pitch;
}
BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
PUSH_DATA (push, nblocksx * cpp);
PUSH_DATA (push, line_count);
BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
PUSH_DATA (push, exec);
height -= line_count;
sy += line_count;
dy += line_count;
}
nouveau_bufctx_reset(bctx, 0);
}
static void
nve4_m2mf_transfer_rect(struct nvc0_context *nvc0,
const struct nv50_m2mf_rect *dst,
const struct nv50_m2mf_rect *src,
uint32_t nblocksx, uint32_t nblocksy)
{
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nouveau_bufctx *bctx = nvc0->bufctx;
uint32_t exec;
uint32_t src_base = src->base;
uint32_t dst_base = dst->base;
const int cpp = dst->cpp;
assert(dst->cpp == src->cpp);
nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
nouveau_pushbuf_bufctx(push, bctx);
nouveau_pushbuf_validate(push);
exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */;
if (!nouveau_bo_memtype(dst->bo)) {
assert(!dst->z);
dst_base += dst->y * dst->pitch + dst->x * cpp;
exec |= 0x100; /* DST_MODE_2D_LINEAR */
}
if (!nouveau_bo_memtype(src->bo)) {
assert(!src->z);
src_base += src->y * src->pitch + src->x * cpp;
exec |= 0x080; /* SRC_MODE_2D_LINEAR */
}
BEGIN_NVC0(push, SUBC_COPY(0x070c), 6);
PUSH_DATA (push, 0x1000 | dst->tile_mode);
PUSH_DATA (push, dst->pitch);
PUSH_DATA (push, dst->height);
PUSH_DATA (push, dst->depth);
PUSH_DATA (push, dst->z);
PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp));
BEGIN_NVC0(push, SUBC_COPY(0x0728), 6);
PUSH_DATA (push, 0x1000 | src->tile_mode);
PUSH_DATA (push, src->pitch);
PUSH_DATA (push, src->height);
PUSH_DATA (push, src->depth);
PUSH_DATA (push, src->z);
PUSH_DATA (push, (src->y << 16) | (src->x * cpp));
BEGIN_NVC0(push, SUBC_COPY(0x0400), 8);
PUSH_DATAh(push, src->bo->offset + src_base);
PUSH_DATA (push, src->bo->offset + src_base);
PUSH_DATAh(push, dst->bo->offset + dst_base);
PUSH_DATA (push, dst->bo->offset + dst_base);
PUSH_DATA (push, src->pitch);
PUSH_DATA (push, dst->pitch);
PUSH_DATA (push, nblocksx * cpp);
PUSH_DATA (push, nblocksy);
BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
PUSH_DATA (push, exec);
nouveau_bufctx_reset(bctx, 0);
}
void
nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
{
struct push_context ctx;
unsigned i, index_size;
unsigned inst_count = info->instance_count;
unsigned vert_count = info->count;
unsigned prim;
nvc0_push_context_init(nvc0, &ctx);
nvc0_vertex_configure_translate(nvc0, info->index_bias);
if (unlikely(ctx.edgeflag.enabled))
nvc0_push_map_edgeflag(&ctx, nvc0, info->index_bias);
ctx.prim_restart = info->primitive_restart;
ctx.restart_index = info->restart_index;
if (info->indexed) {
nvc0_push_map_idxbuf(&ctx, nvc0);
index_size = nvc0->idxbuf.index_size;
if (info->primitive_restart) {
BEGIN_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
PUSH_DATA (ctx.push, 1);
PUSH_DATA (ctx.push, info->restart_index);
} else
if (nvc0->state.prim_restart) {
IMMED_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
}
nvc0->state.prim_restart = info->primitive_restart;
} else {
if (unlikely(info->count_from_stream_output)) {
struct pipe_context *pipe = &nvc0->base.pipe;
struct nvc0_so_target *targ;
targ = nvc0_so_target(info->count_from_stream_output);
pipe->get_query_result(pipe, targ->pq, TRUE, (void *)&vert_count);
vert_count /= targ->stride;
}
ctx.idxbuf = NULL; /* shut up warnings */
index_size = 0;
}
ctx.instance_id = info->start_instance;
prim = nvc0_prim_gl(info->mode);
do {
PUSH_SPACE(ctx.push, 9);
ctx.dest = nvc0_push_setup_vertex_array(nvc0, vert_count);
if (unlikely(!ctx.dest))
break;
if (unlikely(ctx.need_vertex_id))
nvc0_push_upload_vertex_ids(&ctx, nvc0, info);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_BEGIN_GL), 1);
PUSH_DATA (ctx.push, prim);
switch (index_size) {
case 1:
disp_vertices_i08(&ctx, info->start, vert_count);
break;
case 2:
disp_vertices_i16(&ctx, info->start, vert_count);
break;
case 4:
disp_vertices_i32(&ctx, info->start, vert_count);
break;
default:
assert(index_size == 0);
disp_vertices_seq(&ctx, info->start, vert_count);
break;
}
PUSH_SPACE(ctx.push, 1);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_END_GL), 0);
if (--inst_count) {
prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
++ctx.instance_id;
}
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX_TMP);
nouveau_scratch_done(&nvc0->base);
} while (inst_count);
/* reset state and unmap buffers (no-op) */
if (unlikely(!ctx.edgeflag.value)) {
PUSH_SPACE(ctx.push, 1);
IMMED_NVC0(ctx.push, NVC0_3D(EDGEFLAG), 1);
}
if (unlikely(ctx.need_vertex_id)) {
PUSH_SPACE(ctx.push, 4);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ID_REPLACE), 0);
BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_ATTRIB_FORMAT(1)), 1);
PUSH_DATA (ctx.push,
NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST |
NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT |
NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32);
IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 0);
}
if (info->indexed)
nouveau_resource_unmap(nv04_resource(nvc0->idxbuf.buffer));
for (i = 0; i < nvc0->num_vtxbufs; ++i)
nouveau_resource_unmap(nv04_resource(nvc0->vtxbuf[i].buffer));
NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_fallback_count, 1);
}
void
nvc0_launch_grid(struct pipe_context *pipe, const struct pipe_grid_info *info)
{
struct nvc0_context *nvc0 = nvc0_context(pipe);
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
struct nvc0_program *cp = nvc0->compprog;
int ret;
ret = !nvc0_state_validate_cp(nvc0, ~0);
if (ret) {
NOUVEAU_ERR("Failed to launch grid !\n");
return;
}
nvc0_compute_upload_input(nvc0, info);
BEGIN_NVC0(push, NVC0_CP(CP_START_ID), 1);
PUSH_DATA (push, nvc0_program_symbol_offset(cp, info->pc));
BEGIN_NVC0(push, NVC0_CP(LOCAL_POS_ALLOC), 3);
PUSH_DATA (push, (cp->hdr[1] & 0xfffff0) + align(cp->cp.lmem_size, 0x10));
PUSH_DATA (push, 0);
PUSH_DATA (push, 0x800); /* WARP_CSTACK_SIZE */
BEGIN_NVC0(push, NVC0_CP(SHARED_SIZE), 3);
PUSH_DATA (push, align(cp->cp.smem_size, 0x100));
PUSH_DATA (push, info->block[0] * info->block[1] * info->block[2]);
PUSH_DATA (push, cp->num_barriers);
BEGIN_NVC0(push, NVC0_CP(CP_GPR_ALLOC), 1);
PUSH_DATA (push, cp->num_gprs);
/* launch preliminary setup */
BEGIN_NVC0(push, NVC0_CP(GRIDID), 1);
PUSH_DATA (push, 0x1);
BEGIN_NVC0(push, SUBC_CP(0x036c), 1);
PUSH_DATA (push, 0);
BEGIN_NVC0(push, NVC0_CP(FLUSH), 1);
PUSH_DATA (push, NVC0_COMPUTE_FLUSH_GLOBAL | NVC0_COMPUTE_FLUSH_UNK8);
/* block setup */
BEGIN_NVC0(push, NVC0_CP(BLOCKDIM_YX), 2);
PUSH_DATA (push, (info->block[1] << 16) | info->block[0]);
PUSH_DATA (push, info->block[2]);
if (unlikely(info->indirect)) {
struct nv04_resource *res = nv04_resource(info->indirect);
uint32_t offset = res->offset + info->indirect_offset;
unsigned macro = NVC0_CP_MACRO_LAUNCH_GRID_INDIRECT;
nouveau_pushbuf_space(push, 16, 0, 1);
PUSH_REFN(push, res->bo, NOUVEAU_BO_RD | res->domain);
PUSH_DATA(push, NVC0_FIFO_PKHDR_1I(1, macro, 3));
nouveau_pushbuf_data(push, res->bo, offset,
NVC0_IB_ENTRY_1_NO_PREFETCH | 3 * 4);
} else {
/* grid setup */
BEGIN_NVC0(push, NVC0_CP(GRIDDIM_YX), 2);
PUSH_DATA (push, (info->grid[1] << 16) | info->grid[0]);
PUSH_DATA (push, info->grid[2]);
/* kernel launching */
BEGIN_NVC0(push, NVC0_CP(COMPUTE_BEGIN), 1);
PUSH_DATA (push, 0);
BEGIN_NVC0(push, SUBC_CP(0x0a08), 1);
PUSH_DATA (push, 0);
BEGIN_NVC0(push, NVC0_CP(LAUNCH), 1);
PUSH_DATA (push, 0x1000);
BEGIN_NVC0(push, NVC0_CP(COMPUTE_END), 1);
PUSH_DATA (push, 0);
BEGIN_NVC0(push, SUBC_CP(0x0360), 1);
PUSH_DATA (push, 0x1);
}
/* TODO: Not sure if this is really necessary. */
nvc0_compute_invalidate_surfaces(nvc0, 5);
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_SUF);
nvc0->dirty_cp |= NVC0_NEW_CP_SURFACES;
nvc0->images_dirty[5] |= nvc0->images_valid[5];
}
static int
nv50_invalidate_resource_storage(struct nouveau_context *ctx,
struct pipe_resource *res,
int ref)
{
struct nv50_context *nv50 = nv50_context(&ctx->pipe);
unsigned bind = res->bind ? res->bind : PIPE_BIND_VERTEX_BUFFER;
unsigned s, i;
if (bind & PIPE_BIND_RENDER_TARGET) {
assert(nv50->framebuffer.nr_cbufs <= PIPE_MAX_COLOR_BUFS);
for (i = 0; i < nv50->framebuffer.nr_cbufs; ++i) {
if (nv50->framebuffer.cbufs[i] &&
nv50->framebuffer.cbufs[i]->texture == res) {
nv50->dirty_3d |= NV50_NEW_3D_FRAMEBUFFER;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_3D_FB);
if (!--ref)
return ref;
}
}
}
if (bind & PIPE_BIND_DEPTH_STENCIL) {
if (nv50->framebuffer.zsbuf &&
nv50->framebuffer.zsbuf->texture == res) {
nv50->dirty_3d |= NV50_NEW_3D_FRAMEBUFFER;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_3D_FB);
if (!--ref)
return ref;
}
}
if (bind & (PIPE_BIND_VERTEX_BUFFER |
PIPE_BIND_INDEX_BUFFER |
PIPE_BIND_CONSTANT_BUFFER |
PIPE_BIND_STREAM_OUTPUT |
PIPE_BIND_SAMPLER_VIEW)) {
assert(nv50->num_vtxbufs <= PIPE_MAX_ATTRIBS);
for (i = 0; i < nv50->num_vtxbufs; ++i) {
if (nv50->vtxbuf[i].buffer == res) {
nv50->dirty_3d |= NV50_NEW_3D_ARRAYS;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_3D_VERTEX);
if (!--ref)
return ref;
}
}
if (nv50->idxbuf.buffer == res) {
/* Just rebind to the bufctx as there is no separate dirty bit */
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_3D_INDEX);
BCTX_REFN(nv50->bufctx_3d, 3D_INDEX, nv04_resource(res), RD);
if (!--ref)
return ref;
}
for (s = 0; s < 3; ++s) {
assert(nv50->num_textures[s] <= PIPE_MAX_SAMPLERS);
for (i = 0; i < nv50->num_textures[s]; ++i) {
if (nv50->textures[s][i] &&
nv50->textures[s][i]->texture == res) {
nv50->dirty_3d |= NV50_NEW_3D_TEXTURES;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_3D_TEXTURES);
if (!--ref)
return ref;
}
}
}
for (s = 0; s < 3; ++s) {
for (i = 0; i < NV50_MAX_PIPE_CONSTBUFS; ++i) {
if (!(nv50->constbuf_valid[s] & (1 << i)))
continue;
if (!nv50->constbuf[s][i].user &&
nv50->constbuf[s][i].u.buf == res) {
nv50->dirty_3d |= NV50_NEW_3D_CONSTBUF;
nv50->constbuf_dirty[s] |= 1 << i;
nouveau_bufctx_reset(nv50->bufctx_3d, NV50_BIND_3D_CB(s, i));
if (!--ref)
return ref;
}
}
}
}
return ref;
}
static int
nvc0_invalidate_resource_storage(struct nouveau_context *ctx,
struct pipe_resource *res,
int ref)
{
struct nvc0_context *nvc0 = nvc0_context(&ctx->pipe);
unsigned s, i;
if (res->bind & PIPE_BIND_RENDER_TARGET) {
for (i = 0; i < nvc0->framebuffer.nr_cbufs; ++i) {
if (nvc0->framebuffer.cbufs[i] &&
nvc0->framebuffer.cbufs[i]->texture == res) {
nvc0->dirty_3d |= NVC0_NEW_3D_FRAMEBUFFER;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_FB);
if (!--ref)
return ref;
}
}
}
if (res->bind & PIPE_BIND_DEPTH_STENCIL) {
if (nvc0->framebuffer.zsbuf &&
nvc0->framebuffer.zsbuf->texture == res) {
nvc0->dirty_3d |= NVC0_NEW_3D_FRAMEBUFFER;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_FB);
if (!--ref)
return ref;
}
}
if (res->target == PIPE_BUFFER) {
for (i = 0; i < nvc0->num_vtxbufs; ++i) {
if (nvc0->vtxbuf[i].buffer == res) {
nvc0->dirty_3d |= NVC0_NEW_3D_ARRAYS;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_VTX);
if (!--ref)
return ref;
}
}
if (nvc0->idxbuf.buffer == res) {
nvc0->dirty_3d |= NVC0_NEW_3D_IDXBUF;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_IDX);
if (!--ref)
return ref;
}
for (s = 0; s < 6; ++s) {
for (i = 0; i < nvc0->num_textures[s]; ++i) {
if (nvc0->textures[s][i] &&
nvc0->textures[s][i]->texture == res) {
nvc0->textures_dirty[s] |= 1 << i;
if (unlikely(s == 5)) {
nvc0->dirty_cp |= NVC0_NEW_CP_TEXTURES;
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_TEX(i));
} else {
nvc0->dirty_3d |= NVC0_NEW_3D_TEXTURES;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_TEX(s, i));
}
if (!--ref)
return ref;
}
}
}
for (s = 0; s < 6; ++s) {
for (i = 0; i < NVC0_MAX_PIPE_CONSTBUFS; ++i) {
if (!(nvc0->constbuf_valid[s] & (1 << i)))
continue;
if (!nvc0->constbuf[s][i].user &&
nvc0->constbuf[s][i].u.buf == res) {
nvc0->constbuf_dirty[s] |= 1 << i;
if (unlikely(s == 5)) {
nvc0->dirty_cp |= NVC0_NEW_CP_CONSTBUF;
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_CB(i));
} else {
nvc0->dirty_3d |= NVC0_NEW_3D_CONSTBUF;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_CB(s, i));
}
if (!--ref)
return ref;
}
}
}
for (s = 0; s < 6; ++s) {
for (i = 0; i < NVC0_MAX_BUFFERS; ++i) {
if (nvc0->buffers[s][i].buffer == res) {
nvc0->buffers_dirty[s] |= 1 << i;
if (unlikely(s == 5)) {
nvc0->dirty_cp |= NVC0_NEW_CP_BUFFERS;
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_BUF);
} else {
nvc0->dirty_3d |= NVC0_NEW_3D_BUFFERS;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_BUF);
}
if (!--ref)
return ref;
}
}
}
for (s = 0; s < 6; ++s) {
for (i = 0; i < NVC0_MAX_IMAGES; ++i) {
if (nvc0->images[s][i].resource == res) {
nvc0->images_dirty[s] |= 1 << i;
if (unlikely(s == 5)) {
nvc0->dirty_cp |= NVC0_NEW_CP_SURFACES;
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_SUF);
} else {
nvc0->dirty_3d |= NVC0_NEW_3D_SURFACES;
nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_SUF);
}
}
if (!--ref)
return ref;
}
}
}
return ref;
}
static void
nve4_mp_pm_query_end(struct nvc0_context *nvc0, struct nvc0_query *q)
{
struct nvc0_screen *screen = nvc0->screen;
struct pipe_context *pipe = &nvc0->base.pipe;
struct nouveau_pushbuf *push = nvc0->base.pushbuf;
uint32_t mask;
uint32_t input[3];
const uint block[3] = { 32, 4, 1 };
const uint grid[3] = { screen->mp_count, 1, 1 };
unsigned c;
const struct nve4_mp_pm_query_cfg *cfg;
cfg = &nve4_mp_pm_queries[q->type - PIPE_QUERY_DRIVER_SPECIFIC];
if (unlikely(!screen->pm.prog)) {
struct nvc0_program *prog = CALLOC_STRUCT(nvc0_program);
prog->type = PIPE_SHADER_COMPUTE;
prog->translated = TRUE;
prog->num_gprs = 14;
prog->code = (uint32_t *)nve4_read_mp_pm_counters_code;
prog->code_size = sizeof(nve4_read_mp_pm_counters_code);
prog->parm_size = 12;
screen->pm.prog = prog;
}
/* disable all counting */
PUSH_SPACE(push, 8);
for (c = 0; c < 8; ++c)
if (screen->pm.mp_counter[c])
IMMED_NVC0(push, NVE4_COMPUTE(MP_PM_FUNC(c)), 0);
/* release counters for this query */
for (c = 0; c < 8; ++c) {
if (nvc0_query(screen->pm.mp_counter[c]) == q) {
screen->pm.num_mp_pm_active[c / 4]--;
screen->pm.mp_counter[c] = NULL;
}
}
BCTX_REFN_bo(nvc0->bufctx_cp, CP_QUERY, NOUVEAU_BO_GART | NOUVEAU_BO_WR,
q->bo);
PUSH_SPACE(push, 1);
IMMED_NVC0(push, SUBC_COMPUTE(NV50_GRAPH_SERIALIZE), 0);
pipe->bind_compute_state(pipe, screen->pm.prog);
input[0] = (q->bo->offset + q->base);
input[1] = (q->bo->offset + q->base) >> 32;
input[2] = q->sequence;
pipe->launch_grid(pipe, block, grid, 0, input);
nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_QUERY);
/* re-activate other counters */
PUSH_SPACE(push, 16);
mask = 0;
for (c = 0; c < 8; ++c) {
unsigned i;
q = nvc0_query(screen->pm.mp_counter[c]);
if (!q)
continue;
cfg = &nve4_mp_pm_queries[q->type - PIPE_QUERY_DRIVER_SPECIFIC];
for (i = 0; i < cfg->num_counters; ++i) {
if (mask & (1 << q->ctr[i]))
break;
mask |= 1 << q->ctr[i];
BEGIN_NVC0(push, NVE4_COMPUTE(MP_PM_FUNC(q->ctr[i])), 1);
PUSH_DATA (push, (cfg->ctr[i].func << 4) | cfg->ctr[i].mode);
}
}
}
