/**
* adreno_drawctxt_create - create a new adreno draw context
* @device - KGSL device to create the context on
* @pagetable - Pagetable for the context
* @context- Generic KGSL context structure
* @flags - flags for the context (passed from user space)
*
* Create a new draw context for the 3D core. Return 0 on success,
* or error code on failure.
*/
int adreno_drawctxt_create(struct kgsl_device *device,
struct kgsl_pagetable *pagetable,
struct kgsl_context *context, uint32_t flags)
{
struct adreno_context *drawctxt;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return -ENOMEM;
drawctxt->pagetable = pagetable;
drawctxt->bin_base_offset = 0;
drawctxt->id = context->id;
rb->timestamp[context->id] = 0;
if (flags & KGSL_CONTEXT_PREAMBLE)
drawctxt->flags |= CTXT_FLAGS_PREAMBLE;
if (flags & KGSL_CONTEXT_NO_GMEM_ALLOC)
drawctxt->flags |= CTXT_FLAGS_NOGMEMALLOC;
if (flags & KGSL_CONTEXT_PER_CONTEXT_TS)
drawctxt->flags |= CTXT_FLAGS_PER_CONTEXT_TS;
if (flags & KGSL_CONTEXT_USER_GENERATED_TS) {
if (!(flags & KGSL_CONTEXT_PER_CONTEXT_TS)) {
ret = -EINVAL;
goto err;
}
drawctxt->flags |= CTXT_FLAGS_USER_GENERATED_TS;
}
ret = adreno_dev->gpudev->ctxt_create(adreno_dev, drawctxt);
if (ret)
goto err;
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->id, ref_wait_ts),
KGSL_INIT_REFTIMESTAMP);
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->id, ts_cmp_enable), 0);
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->id, soptimestamp), 0);
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->id, eoptimestamp), 0);
context->devctxt = drawctxt;
return 0;
err:
kfree(drawctxt);
return ret;
}
/**
* adreno_drawctxt_invalidate() - Invalidate an adreno draw context
* @device: Pointer to the KGSL device structure for the GPU
* @context: Pointer to the KGSL context structure
*
* Invalidate the context and remove all queued commands and cancel any pending
* waiters
*/
void adreno_drawctxt_invalidate(struct kgsl_device *device,
struct kgsl_context *context)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
trace_adreno_drawctxt_invalidate(drawctxt);
drawctxt->state = ADRENO_CONTEXT_STATE_INVALID;
/* Clear the pending queue */
mutex_lock(&drawctxt->mutex);
/*
* set the timestamp to the last value since the context is invalidated
* and we want the pending events for this context to go away
*/
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
while (drawctxt->cmdqueue_head != drawctxt->cmdqueue_tail) {
struct kgsl_cmdbatch *cmdbatch =
drawctxt->cmdqueue[drawctxt->cmdqueue_head];
drawctxt->cmdqueue_head = (drawctxt->cmdqueue_head + 1) %
ADRENO_CONTEXT_CMDQUEUE_SIZE;
mutex_unlock(&drawctxt->mutex);
kgsl_mutex_lock(&device->mutex, &device->mutex_owner);
kgsl_cancel_events_timestamp(device, &context->events,
cmdbatch->timestamp);
kgsl_mutex_unlock(&device->mutex, &device->mutex_owner);
kgsl_cmdbatch_destroy(cmdbatch);
mutex_lock(&drawctxt->mutex);
}
mutex_unlock(&drawctxt->mutex);
/* Give the bad news to everybody waiting around */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
}
/**
* adreno_context_restore() - generic context restore handler
* @adreno_dev: the device
* @context: the context
*
* Basic context restore handler that writes the context identifier
* to the ringbuffer and issues pagetable switch commands if necessary.
*/
static int adreno_context_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device;
unsigned int cmds[8];
if (adreno_dev == NULL || context == NULL)
return -EINVAL;
device = &adreno_dev->dev;
/* write the context identifier to the ringbuffer */
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
cmds[4] = context->base.id;
/* Flush the UCHE for new context */
cmds[5] = cp_type0_packet(
adreno_getreg(adreno_dev, ADRENO_REG_UCHE_INVALIDATE0), 2);
cmds[6] = 0;
if (adreno_is_a4xx(adreno_dev))
cmds[7] = 0x12;
else if (adreno_is_a3xx(adreno_dev))
cmds[7] = 0x90000000;
return adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, cmds, 8);
}
/**
* adreno_drawctxt_invalidate() - Invalidate an adreno draw context
* @device: Pointer to the KGSL device structure for the GPU
* @context: Pointer to the KGSL context structure
*
* Invalidate the context and remove all queued commands and cancel any pending
* waiters
*/
void adreno_drawctxt_invalidate(struct kgsl_device *device,
struct kgsl_context *context)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
trace_adreno_drawctxt_invalidate(drawctxt);
spin_lock(&drawctxt->lock);
set_bit(KGSL_CONTEXT_PRIV_INVALID, &context->priv);
/*
* set the timestamp to the last value since the context is invalidated
* and we want the pending events for this context to go away
*/
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
/* Get rid of commands still waiting in the queue */
while (drawctxt->cmdqueue_head != drawctxt->cmdqueue_tail) {
struct kgsl_cmdbatch *cmdbatch =
drawctxt->cmdqueue[drawctxt->cmdqueue_head];
drawctxt->cmdqueue_head = (drawctxt->cmdqueue_head + 1) %
ADRENO_CONTEXT_CMDQUEUE_SIZE;
kgsl_cancel_events_timestamp(device, &context->events,
cmdbatch->timestamp);
kgsl_cmdbatch_destroy(cmdbatch);
}
spin_unlock(&drawctxt->lock);
/* Make sure all "retired" events are processed */
kgsl_process_event_group(device, &context->events);
/* Give the bad news to everybody waiting around */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
}
void adreno_drawctxt_invalidate(struct kgsl_device *device,
struct kgsl_context *context)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
trace_adreno_drawctxt_invalidate(drawctxt);
drawctxt->state = ADRENO_CONTEXT_STATE_INVALID;
mutex_lock(&drawctxt->mutex);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
while (drawctxt->cmdqueue_head != drawctxt->cmdqueue_tail) {
struct kgsl_cmdbatch *cmdbatch =
drawctxt->cmdqueue[drawctxt->cmdqueue_head];
drawctxt->cmdqueue_head = (drawctxt->cmdqueue_head + 1) %
ADRENO_CONTEXT_CMDQUEUE_SIZE;
mutex_unlock(&drawctxt->mutex);
mutex_lock(&device->mutex);
kgsl_cancel_events_timestamp(device, context,
cmdbatch->timestamp);
mutex_unlock(&device->mutex);
kgsl_cmdbatch_destroy(cmdbatch);
mutex_lock(&drawctxt->mutex);
}
mutex_unlock(&drawctxt->mutex);
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
}
static void _adreno_context_restore_cpu(struct adreno_ringbuffer *rb,
struct adreno_context *drawctxt)
{
kgsl_sharedmem_writel(rb->device, &(rb->device->memstore),
KGSL_MEMSTORE_RB_OFFSET(rb->device, current_context),
drawctxt ? drawctxt->base.id : 0);
kgsl_sharedmem_writel(rb->device, &(rb->device->memstore),
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
current_context),
drawctxt ? drawctxt->base.id : 0);
}
/**
* adreno_drawctxt_invalidate() - Invalidate an adreno draw context
* @device: Pointer to the KGSL device structure for the GPU
* @context: Pointer to the KGSL context structure
*
* Invalidate the context and remove all queued commands and cancel any pending
* waiters
*/
void adreno_drawctxt_invalidate(struct kgsl_device *device,
struct kgsl_context *context)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
struct kgsl_cmdbatch *list[ADRENO_CONTEXT_CMDQUEUE_SIZE];
int i, count;
trace_adreno_drawctxt_invalidate(drawctxt);
spin_lock(&drawctxt->lock);
set_bit(KGSL_CONTEXT_PRIV_INVALID, &context->priv);
/*
* set the timestamp to the last value since the context is invalidated
* and we want the pending events for this context to go away
*/
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
/* Get rid of commands still waiting in the queue */
count = drawctxt_detach_cmdbatches(drawctxt, list);
spin_unlock(&drawctxt->lock);
for (i = 0; i < count; i++) {
kgsl_cancel_events_timestamp(device, &context->events,
list[i]->timestamp);
kgsl_cmdbatch_destroy(list[i]);
}
/* Make sure all pending events are processed or cancelled */
kgsl_flush_event_group(device, &context->events);
/* Give the bad news to everybody waiting around */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
}
/**
* adreno_context_restore() - generic context restore handler
* @rb: The RB in which context is to be restored
*
* Basic context restore handler that writes the context identifier
* to the ringbuffer and issues pagetable switch commands if necessary.
*/
static void adreno_context_restore(struct adreno_ringbuffer *rb)
{
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_context *drawctxt = rb->drawctxt_active;
unsigned int cmds[11];
int ret;
if (!drawctxt)
return;
/*
* write the context identifier to the ringbuffer, write to both
* the global index and the index of the RB in which the context
* operates. The global values will always be reliable since we
* could be in middle of RB switch in which case the RB value may
* not be accurate
*/
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_RB_OFFSET(rb, current_context);
cmds[4] = drawctxt->base.id;
cmds[5] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[6] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
current_context);
cmds[7] = drawctxt->base.id;
/* Flush the UCHE for new context */
cmds[8] = cp_type0_packet(
adreno_getreg(adreno_dev, ADRENO_REG_UCHE_INVALIDATE0), 2);
cmds[9] = 0;
if (adreno_is_a4xx(adreno_dev))
cmds[10] = 0x12;
else if (adreno_is_a3xx(adreno_dev))
cmds[10] = 0x90000000;
ret = adreno_ringbuffer_issuecmds(rb, KGSL_CMD_FLAGS_NONE, cmds, 11);
if (ret) {
/*
* A failure to submit commands to ringbuffer means RB may
* be full, in this case wait for idle and use CPU
*/
ret = adreno_idle(device);
BUG_ON(ret);
_adreno_context_restore_cpu(rb, drawctxt);
}
}
int adreno_context_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device;
unsigned int cmds[5];
if (adreno_dev == NULL || context == NULL)
return -EINVAL;
device = &adreno_dev->dev;
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
cmds[4] = context->base.id;
return adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, cmds, 5);
}
/**
* adreno_drawctxt_create - create a new adreno draw context
* @device - KGSL device to create the context on
* @pagetable - Pagetable for the context
* @context- Generic KGSL context structure
* @flags - flags for the context (passed from user space)
*
* Create a new draw context for the 3D core. Return 0 on success,
* or error code on failure.
*/
int adreno_drawctxt_create(struct kgsl_device *device,
struct kgsl_pagetable *pagetable,
struct kgsl_context *context, uint32_t flags)
{
struct adreno_context *drawctxt;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return -ENOMEM;
drawctxt->pagetable = pagetable;
drawctxt->bin_base_offset = 0;
drawctxt->id = context->id;
if (flags & KGSL_CONTEXT_PREAMBLE)
drawctxt->flags |= CTXT_FLAGS_PREAMBLE;
if (flags & KGSL_CONTEXT_NO_GMEM_ALLOC)
drawctxt->flags |= CTXT_FLAGS_NOGMEMALLOC;
if (flags & KGSL_CONTEXT_PER_CONTEXT_TS)
drawctxt->flags |= CTXT_FLAGS_PER_CONTEXT_TS;
ret = adreno_dev->gpudev->ctxt_create(adreno_dev, drawctxt);
if (ret)
goto err;
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->id, ref_wait_ts),
KGSL_INIT_REFTIMESTAMP);
context->devctxt = drawctxt;
return 0;
err:
kfree(drawctxt);
return ret;
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
/*
* if the context was not created with per context timestamp
* support, we must use the global timestamp since issueibcmds
* will be returning that one.
*/
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
if (adreno_is_a2xx(adreno_dev))
total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */
total_sizedwords += 2; /* scratchpad ts for recovery */
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
total_sizedwords += 3; /* sop timestamp */
total_sizedwords += 4; /* eop timestamp */
total_sizedwords += 3; /* global timestamp without cache
* flush for non-zero context */
} else {
total_sizedwords += 4; /* global timestamp for recovery*/
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
/* GPU may hang during space allocation, if thats the case the current
* context may have hung the GPU */
if (context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp[context_id];
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
/* always increment the global timestamp. once. */
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
/* HW Workaround for MMU Page fault
* due to memory getting free early before
* GPU completes it.
*/
if (adreno_is_a2xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
/* scratchpad ts for recovery */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
/* start-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* end-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
{
int status;
/*cp_rb_cntl_u cp_rb_cntl; */
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int rb_cntl;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (rb->flags & KGSL_FLAGS_STARTED)
return 0;
if (init_ram)
rb->timestamp[KGSL_MEMSTORE_GLOBAL] = 0;
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
if (adreno_is_a2xx(adreno_dev)) {
adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
/* setup WPTR delay */
adreno_regwrite(device, REG_CP_RB_WPTR_DELAY,
0 /*0x70000010 */);
}
/*setup REG_CP_RB_CNTL */
adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
/*
* The size of the ringbuffer in the hardware is the log2
* representation of the size in quadwords (sizedwords / 2)
*/
cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
/*
* Specify the quadwords to read before updating mem RPTR.
* Like above, pass the log2 representation of the blocksize
* in quadwords.
*/
cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
if (adreno_is_a2xx(adreno_dev)) {
/* WPTR polling */
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
}
/* mem RPTR writebacks */
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
if (adreno_is_a2xx(adreno_dev)) {
/* explicitly clear all cp interrupts */
adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
}
/* setup scratch/timestamp */
adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
soptimestamp));
adreno_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
/* load the CP ucode */
status = adreno_ringbuffer_load_pm4_ucode(device);
if (status != 0)
return status;
/* load the prefetch parser ucode */
status = adreno_ringbuffer_load_pfp_ucode(device);
if (status != 0)
return status;
rb->rptr = 0;
rb->wptr = 0;
/* clear ME_HALT to start micro engine */
adreno_regwrite(device, REG_CP_ME_CNTL, 0);
/* ME init is GPU specific, so jump into the sub-function */
adreno_dev->gpudev->rb_init(adreno_dev, rb);
/* idle device to validate ME INIT */
status = adreno_idle(device);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
return status;
}
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
{
int status;
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int rb_cntl;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (rb->flags & KGSL_FLAGS_STARTED)
return 0;
if (init_ram)
rb->timestamp[KGSL_MEMSTORE_GLOBAL] = 0;
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
if (adreno_is_a2xx(adreno_dev)) {
adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
adreno_regwrite(device, REG_CP_RB_WPTR_DELAY,
0 );
}
adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
if (adreno_is_a2xx(adreno_dev)) {
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
}
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
if (adreno_is_a3xx(adreno_dev)) {
adreno_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_0, 0x63000040);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_1, 0x62000080);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_2, 0x600000CC);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_3, 0x60000108);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_4, 0x64000140);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_5, 0x66000400);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_6, 0x65000700);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_7, 0x610007D8);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_8, 0x620007E0);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_9, 0x61001178);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_A, 0x64001180);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_B, 0x60003300);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_C, 0x6B00C000);
}
if (adreno_is_a2xx(adreno_dev)) {
adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
}
adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
soptimestamp));
adreno_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
status = adreno_ringbuffer_load_pm4_ucode(device);
if (status != 0)
return status;
status = adreno_ringbuffer_load_pfp_ucode(device);
if (status != 0)
return status;
if (adreno_is_a305(adreno_dev) || adreno_is_a320(adreno_dev))
adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000E0602);
rb->rptr = 0;
rb->wptr = 0;
adreno_regwrite(device, REG_CP_ME_CNTL, 0);
adreno_dev->gpudev->rb_init(adreno_dev, rb);
status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
return status;
}
/**
* adreno_drawctxt_detach(): detach a context from the GPU
* @context: Generic KGSL context container for the context
*
*/
int adreno_drawctxt_detach(struct kgsl_context *context)
{
struct kgsl_device *device;
struct adreno_device *adreno_dev;
struct adreno_context *drawctxt;
struct adreno_ringbuffer *rb;
int ret;
if (context == NULL)
return 0;
device = context->device;
adreno_dev = ADRENO_DEVICE(device);
drawctxt = ADRENO_CONTEXT(context);
rb = drawctxt->rb;
/* deactivate context */
if (rb->drawctxt_active == drawctxt)
adreno_drawctxt_switch(adreno_dev, rb, NULL, 0);
spin_lock(&drawctxt->lock);
while (drawctxt->cmdqueue_head != drawctxt->cmdqueue_tail) {
struct kgsl_cmdbatch *cmdbatch =
drawctxt->cmdqueue[drawctxt->cmdqueue_head];
drawctxt->cmdqueue_head = (drawctxt->cmdqueue_head + 1) %
ADRENO_CONTEXT_CMDQUEUE_SIZE;
spin_unlock(&drawctxt->lock);
/*
* If the context is deteached while we are waiting for
* the next command in GFT SKIP CMD, print the context
* detached status here.
*/
adreno_fault_skipcmd_detached(device, drawctxt, cmdbatch);
/*
* Don't hold the drawctxt mutex while the cmdbatch is being
* destroyed because the cmdbatch destroy takes the device
* mutex and the world falls in on itself
*/
kgsl_cmdbatch_destroy(cmdbatch);
spin_lock(&drawctxt->lock);
}
spin_unlock(&drawctxt->lock);
/*
* internal_timestamp is set in adreno_ringbuffer_addcmds,
* which holds the device mutex. The entire context destroy
* process requires the device mutex as well. But lets
* make sure we notice if the locking changes.
*/
BUG_ON(!mutex_is_locked(&device->mutex));
/*
* Wait for the last global timestamp to pass before continuing.
* The maxumum wait time is 30s, some large IB's can take longer
* than 10s and if hang happens then the time for the context's
* commands to retire will be greater than 10s. 30s should be sufficient
* time to wait for the commands even if a hang happens.
*/
ret = adreno_drawctxt_wait_rb(adreno_dev, context,
drawctxt->internal_timestamp, 30 * 1000);
/*
* If the wait for global fails due to timeout then nothing after this
* point is likely to work very well - BUG_ON() so we can take advantage
* of the debug tools to figure out what the h - e - double hockey
* sticks happened. If EAGAIN error is returned then recovery will kick
* in and there will be no more commands in the RB pipe from this
* context which is waht we are waiting for, so ignore -EAGAIN error
*/
if (-EAGAIN == ret)
ret = 0;
BUG_ON(ret);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
adreno_profile_process_results(adreno_dev);
/* wake threads waiting to submit commands from this context */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
return ret;
}
/**
* adreno_drawctxt_create - create a new adreno draw context
* @dev_priv: the owner of the context
* @flags: flags for the context (passed from user space)
*
* Create and return a new draw context for the 3D core.
*/
struct kgsl_context *
adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
uint32_t *flags)
{
struct adreno_context *drawctxt;
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
int ret;
unsigned long local;
local = *flags & (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
KGSL_CONTEXT_PER_CONTEXT_TS |
KGSL_CONTEXT_USER_GENERATED_TS |
KGSL_CONTEXT_NO_FAULT_TOLERANCE |
KGSL_CONTEXT_CTX_SWITCH |
KGSL_CONTEXT_PRIORITY_MASK |
KGSL_CONTEXT_TYPE_MASK |
KGSL_CONTEXT_PWR_CONSTRAINT |
KGSL_CONTEXT_IFH_NOP |
KGSL_CONTEXT_SECURE);
/* Check for errors before trying to initialize */
/* We no longer support legacy context switching */
if ((local & KGSL_CONTEXT_PREAMBLE) == 0 ||
(local & KGSL_CONTEXT_NO_GMEM_ALLOC) == 0) {
KGSL_DEV_ERR_ONCE(device,
"legacy context switch not supported\n");
return ERR_PTR(-EINVAL);
}
/* Make sure that our target can support secure contexts if requested */
if (!kgsl_mmu_is_secured(&dev_priv->device->mmu) &&
(local & KGSL_CONTEXT_SECURE)) {
KGSL_DEV_ERR_ONCE(device, "Secure context not supported\n");
return ERR_PTR(-EINVAL);
}
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return ERR_PTR(-ENOMEM);
ret = kgsl_context_init(dev_priv, &drawctxt->base);
if (ret != 0) {
kfree(drawctxt);
return ERR_PTR(ret);
}
drawctxt->timestamp = 0;
drawctxt->base.flags = local;
/* Always enable per-context timestamps */
drawctxt->base.flags |= KGSL_CONTEXT_PER_CONTEXT_TS;
drawctxt->type = (drawctxt->base.flags & KGSL_CONTEXT_TYPE_MASK)
>> KGSL_CONTEXT_TYPE_SHIFT;
spin_lock_init(&drawctxt->lock);
init_waitqueue_head(&drawctxt->wq);
init_waitqueue_head(&drawctxt->waiting);
/* Set the context priority */
_set_context_priority(drawctxt);
/* set the context ringbuffer */
drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt);
/*
* Set up the plist node for the dispatcher. Insert the node into the
* drawctxt pending list based on priority.
*/
plist_node_init(&drawctxt->pending, drawctxt->base.priority);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
0);
adreno_context_debugfs_init(ADRENO_DEVICE(device), drawctxt);
/* copy back whatever flags we dediced were valid */
*flags = drawctxt->base.flags;
return &drawctxt->base;
}
struct kgsl_context *
adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
uint32_t *flags)
{
struct adreno_context *drawctxt;
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return ERR_PTR(-ENOMEM);
ret = kgsl_context_init(dev_priv, &drawctxt->base);
if (ret != 0) {
kfree(drawctxt);
return ERR_PTR(ret);
}
drawctxt->bin_base_offset = 0;
drawctxt->timestamp = 0;
drawctxt->base.flags = *flags & (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
KGSL_CONTEXT_PER_CONTEXT_TS |
KGSL_CONTEXT_USER_GENERATED_TS |
KGSL_CONTEXT_NO_FAULT_TOLERANCE |
KGSL_CONTEXT_TYPE_MASK);
drawctxt->base.flags |= KGSL_CONTEXT_PER_CONTEXT_TS;
drawctxt->type = (drawctxt->base.flags & KGSL_CONTEXT_TYPE_MASK)
>> KGSL_CONTEXT_TYPE_SHIFT;
mutex_init(&drawctxt->mutex);
init_waitqueue_head(&drawctxt->wq);
init_waitqueue_head(&drawctxt->waiting);
plist_node_init(&drawctxt->pending, ADRENO_CONTEXT_DEFAULT_PRIORITY);
if (adreno_dev->gpudev->ctxt_create) {
ret = adreno_dev->gpudev->ctxt_create(adreno_dev, drawctxt);
if (ret)
goto err;
} else if ((drawctxt->base.flags & KGSL_CONTEXT_PREAMBLE) == 0 ||
(drawctxt->base.flags & KGSL_CONTEXT_NO_GMEM_ALLOC) == 0) {
KGSL_DEV_ERR_ONCE(device,
"legacy context switch not supported\n");
ret = -EINVAL;
goto err;
} else {
drawctxt->ops = &adreno_preamble_ctx_ops;
}
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
0);
*flags = drawctxt->base.flags;
return &drawctxt->base;
err:
kgsl_context_detach(&drawctxt->base);
return ERR_PTR(ret);
}
static void a2xx_cp_intrcallback(struct kgsl_device *device)
{
unsigned int status = 0, num_reads = 0, master_status = 0;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
int i;
adreno_regread(device, REG_MASTER_INT_SIGNAL, &master_status);
while (!status && (num_reads < VALID_STATUS_COUNT_MAX) &&
(master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) {
adreno_regread(device, REG_CP_INT_STATUS, &status);
adreno_regread(device, REG_MASTER_INT_SIGNAL,
&master_status);
num_reads++;
}
if (num_reads > 1)
KGSL_DRV_WARN(device,
"Looped %d times to read REG_CP_INT_STATUS\n",
num_reads);
trace_kgsl_a2xx_irq_status(device, master_status, status);
if (!status) {
if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) {
KGSL_DRV_WARN(device, "Unable to read CP_INT_STATUS\n");
wake_up_interruptible_all(&device->wait_queue);
} else
KGSL_DRV_WARN(device, "Spurious interrput detected\n");
return;
}
if (status & CP_INT_CNTL__RB_INT_MASK) {
unsigned int context_id;
kgsl_sharedmem_readl(&device->memstore,
&context_id,
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
current_context));
if (context_id < KGSL_MEMSTORE_MAX) {
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(context_id,
ts_cmp_enable), 0);
kgsl_sharedmem_writel(&device->memstore,
KGSL_MEMSTORE_OFFSET(
KGSL_MEMSTORE_GLOBAL,
ts_cmp_enable), 0);
wmb();
}
KGSL_CMD_WARN(rb->device, "ringbuffer rb interrupt\n");
}
for (i = 0; i < ARRAY_SIZE(kgsl_cp_error_irqs); i++) {
if (status & kgsl_cp_error_irqs[i].mask) {
KGSL_CMD_CRIT(rb->device, "%s\n",
kgsl_cp_error_irqs[i].message);
kgsl_pwrctrl_irq(rb->device, KGSL_PWRFLAGS_OFF);
}
}
status &= CP_INT_MASK;
adreno_regwrite(device, REG_CP_INT_ACK, status);
if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) {
KGSL_CMD_WARN(rb->device, "ringbuffer ib1/rb interrupt\n");
queue_work(device->work_queue, &device->ts_expired_ws);
wake_up_interruptible_all(&device->wait_queue);
atomic_notifier_call_chain(&(device->ts_notifier_list),
device->id,
NULL);
}
}
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
{
int status;
/*cp_rb_cntl_u cp_rb_cntl; */
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int rb_cntl;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (rb->flags & KGSL_FLAGS_STARTED)
return 0;
if (init_ram)
rb->timestamp[KGSL_MEMSTORE_GLOBAL] = 0;
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
if (adreno_is_a2xx(adreno_dev)) {
adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
/* setup WPTR delay */
adreno_regwrite(device, REG_CP_RB_WPTR_DELAY,
0 /*0x70000010 */);
}
/*setup REG_CP_RB_CNTL */
adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
/*
* The size of the ringbuffer in the hardware is the log2
* representation of the size in quadwords (sizedwords / 2)
*/
cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
/*
* Specify the quadwords to read before updating mem RPTR.
* Like above, pass the log2 representation of the blocksize
* in quadwords.
*/
cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
if (adreno_is_a2xx(adreno_dev)) {
/* WPTR polling */
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
}
/* mem RPTR writebacks */
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
if (adreno_is_a3xx(adreno_dev)) {
/* enable access protection to privileged registers */
adreno_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007);
/* RBBM registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_0, 0x63000040);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_1, 0x62000080);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_2, 0x600000CC);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_3, 0x60000108);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_4, 0x64000140);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_5, 0x66000400);
/* CP registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_6, 0x65000700);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_7, 0x610007D8);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_8, 0x620007E0);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_9, 0x61001178);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_A, 0x64001180);
/* RB registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_B, 0x60003300);
/* VBIF registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_C, 0x6B00C000);
}
if (adreno_is_a2xx(adreno_dev)) {
/* explicitly clear all cp interrupts */
adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
}
/* setup scratch/timestamp */
adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
soptimestamp));
adreno_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
/* load the CP ucode */
status = adreno_ringbuffer_load_pm4_ucode(device);
if (status != 0)
return status;
/* load the prefetch parser ucode */
status = adreno_ringbuffer_load_pfp_ucode(device);
if (status != 0)
return status;
if (adreno_is_a305(adreno_dev) || adreno_is_a320(adreno_dev))
adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000F0602);
rb->rptr = 0;
rb->wptr = 0;
/* clear ME_HALT to start micro engine */
adreno_regwrite(device, REG_CP_ME_CNTL, 0);
/* ME init is GPU specific, so jump into the sub-function */
adreno_dev->gpudev->rb_init(adreno_dev, rb);
/* idle device to validate ME INIT */
status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
return status;
}
/**
* adreno_drawctxt_detach(): detach a context from the GPU
* @context: Generic KGSL context container for the context
*
*/
int adreno_drawctxt_detach(struct kgsl_context *context)
{
struct kgsl_device *device;
struct adreno_device *adreno_dev;
struct adreno_context *drawctxt;
int ret;
if (context == NULL)
return 0;
device = context->device;
adreno_dev = ADRENO_DEVICE(device);
drawctxt = ADRENO_CONTEXT(context);
/* deactivate context */
if (adreno_dev->drawctxt_active == drawctxt)
adreno_drawctxt_switch(adreno_dev, NULL, 0);
mutex_lock(&drawctxt->mutex);
while (drawctxt->cmdqueue_head != drawctxt->cmdqueue_tail) {
struct kgsl_cmdbatch *cmdbatch =
drawctxt->cmdqueue[drawctxt->cmdqueue_head];
drawctxt->cmdqueue_head = (drawctxt->cmdqueue_head + 1) %
ADRENO_CONTEXT_CMDQUEUE_SIZE;
mutex_unlock(&drawctxt->mutex);
/*
* Don't hold the drawctxt mutex while the cmdbatch is being
* destroyed because the cmdbatch destroy takes the device
* mutex and the world falls in on itself
*/
kgsl_cmdbatch_destroy(cmdbatch);
mutex_lock(&drawctxt->mutex);
}
mutex_unlock(&drawctxt->mutex);
/*
* internal_timestamp is set in adreno_ringbuffer_addcmds,
* which holds the device mutex. The entire context destroy
* process requires the device mutex as well. But lets
* make sure we notice if the locking changes.
*/
BUG_ON(!mutex_is_locked(&device->mutex));
/* Wait for the last global timestamp to pass before continuing */
ret = adreno_drawctxt_wait_global(adreno_dev, context,
drawctxt->internal_timestamp, 10 * 1000);
/*
* If the wait for global fails then nothing after this point is likely
* to work very well - BUG_ON() so we can take advantage of the debug
* tools to figure out what the h - e - double hockey sticks happened
*/
BUG_ON(ret);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
adreno_profile_process_results(device);
/* wake threads waiting to submit commands from this context */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
return ret;
}
int adreno_drawctxt_detach(struct kgsl_context *context)
{
struct kgsl_device *device;
struct adreno_device *adreno_dev;
struct adreno_context *drawctxt;
int ret;
if (context == NULL)
return 0;
device = context->device;
adreno_dev = ADRENO_DEVICE(device);
drawctxt = ADRENO_CONTEXT(context);
if (adreno_dev->drawctxt_active == drawctxt)
adreno_drawctxt_switch(adreno_dev, NULL, 0);
mutex_lock(&drawctxt->mutex);
while (drawctxt->cmdqueue_head != drawctxt->cmdqueue_tail) {
struct kgsl_cmdbatch *cmdbatch =
drawctxt->cmdqueue[drawctxt->cmdqueue_head];
drawctxt->cmdqueue_head = (drawctxt->cmdqueue_head + 1) %
ADRENO_CONTEXT_CMDQUEUE_SIZE;
mutex_unlock(&drawctxt->mutex);
kgsl_cmdbatch_destroy(cmdbatch);
mutex_lock(&drawctxt->mutex);
}
mutex_unlock(&drawctxt->mutex);
BUG_ON(!mutex_is_locked(&device->mutex));
ret = adreno_drawctxt_wait_global(adreno_dev, context,
drawctxt->internal_timestamp, 10 * 1000);
BUG_ON(ret);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
adreno_profile_process_results(device);
if (drawctxt->ops && drawctxt->ops->detach)
drawctxt->ops->detach(drawctxt);
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
return ret;
}
static int kgsl_iommu_fault_handler(struct iommu_domain *domain,
struct device *dev, unsigned long addr, int flags)
{
int ret = 0;
struct kgsl_mmu *mmu;
struct kgsl_iommu *iommu;
struct kgsl_iommu_unit *iommu_unit;
struct kgsl_iommu_device *iommu_dev;
unsigned int ptbase, fsr;
struct kgsl_device *device;
struct adreno_device *adreno_dev;
unsigned int no_page_fault_log = 0;
unsigned int curr_context_id = 0;
unsigned int curr_global_ts = 0;
static struct adreno_context *curr_context;
static struct kgsl_context *context;
ret = get_iommu_unit(dev, &mmu, &iommu_unit);
if (ret)
goto done;
iommu_dev = get_iommu_device(iommu_unit, dev);
if (!iommu_dev) {
KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev);
ret = -ENOSYS;
goto done;
}
iommu = mmu->priv;
device = mmu->device;
adreno_dev = ADRENO_DEVICE(device);
ptbase = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit,
iommu_dev->ctx_id, TTBR0);
fsr = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit,
iommu_dev->ctx_id, FSR);
if (adreno_dev->ft_pf_policy & KGSL_FT_PAGEFAULT_LOG_ONE_PER_PAGE)
no_page_fault_log = kgsl_mmu_log_fault_addr(mmu, ptbase, addr);
if (!no_page_fault_log) {
KGSL_MEM_CRIT(iommu_dev->kgsldev,
"GPU PAGE FAULT: addr = %lX pid = %d\n",
addr, kgsl_mmu_get_ptname_from_ptbase(mmu, ptbase));
KGSL_MEM_CRIT(iommu_dev->kgsldev, "context = %d FSR = %X\n",
iommu_dev->ctx_id, fsr);
}
mmu->fault = 1;
iommu_dev->fault = 1;
kgsl_sharedmem_readl(&device->memstore, &curr_context_id,
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context));
context = idr_find(&device->context_idr, curr_context_id);
if (context != NULL)
curr_context = context->devctxt;
kgsl_sharedmem_readl(&device->memstore, &curr_global_ts,
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, eoptimestamp));
/*
* Store pagefault's timestamp and ib1 addr in context,
* this information is used in GFT
*/
curr_context->pagefault = 1;
curr_context->pagefault_ts = curr_global_ts;
trace_kgsl_mmu_pagefault(iommu_dev->kgsldev, addr,
kgsl_mmu_get_ptname_from_ptbase(mmu, ptbase), 0);
/*
* We do not want the h/w to resume fetching data from an iommu unit
* that has faulted, this is better for debugging as it will stall
* the GPU and trigger a snapshot. To stall the transaction return
* EBUSY error.
*/
if (adreno_dev->ft_pf_policy & KGSL_FT_PAGEFAULT_GPUHALT_ENABLE)
ret = -EBUSY;
done:
return ret;
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += 2;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
total_sizedwords += 2;
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
total_sizedwords += 3;
total_sizedwords += 4;
total_sizedwords += 3;
} else {
total_sizedwords += 4;
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
if (context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp[context_id];
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (adreno_is_a3xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
/**
* adreno_drawctxt_create - create a new adreno draw context
* @dev_priv: the owner of the context
* @flags: flags for the context (passed from user space)
*
* Create and return a new draw context for the 3D core.
*/
struct kgsl_context *
adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
uint32_t *flags)
{
struct adreno_context *drawctxt;
struct kgsl_device *device = dev_priv->device;
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return ERR_PTR(-ENOMEM);
ret = kgsl_context_init(dev_priv, &drawctxt->base);
if (ret != 0) {
kfree(drawctxt);
return ERR_PTR(ret);
}
drawctxt->timestamp = 0;
drawctxt->base.flags = *flags & (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
KGSL_CONTEXT_PER_CONTEXT_TS |
KGSL_CONTEXT_USER_GENERATED_TS |
KGSL_CONTEXT_NO_FAULT_TOLERANCE |
KGSL_CONTEXT_CTX_SWITCH |
KGSL_CONTEXT_PRIORITY_MASK |
KGSL_CONTEXT_TYPE_MASK |
KGSL_CONTEXT_PWR_CONSTRAINT);
/* Always enable per-context timestamps */
drawctxt->base.flags |= KGSL_CONTEXT_PER_CONTEXT_TS;
drawctxt->type = (drawctxt->base.flags & KGSL_CONTEXT_TYPE_MASK)
>> KGSL_CONTEXT_TYPE_SHIFT;
mutex_init(&drawctxt->mutex);
init_waitqueue_head(&drawctxt->wq);
init_waitqueue_head(&drawctxt->waiting);
/* Set the context priority */
_set_context_priority(drawctxt);
/*
* Set up the plist node for the dispatcher. Insert the node into the
* drawctxt pending list based on priority.
*/
plist_node_init(&drawctxt->pending, drawctxt->base.priority);
if ((drawctxt->base.flags & KGSL_CONTEXT_PREAMBLE) == 0 ||
(drawctxt->base.flags & KGSL_CONTEXT_NO_GMEM_ALLOC) == 0) {
KGSL_DEV_ERR_ONCE(device,
"legacy context switch not supported\n");
ret = -EINVAL;
goto err;
}
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
0);
/* copy back whatever flags we dediced were valid */
*flags = drawctxt->base.flags;
return &drawctxt->base;
err:
kgsl_context_detach(&drawctxt->base);
return ERR_PTR(ret);
}
/**
* adreno_drawctxt_create - create a new adreno draw context
* @dev_priv: the owner of the context
* @flags: flags for the context (passed from user space)
*
* Create and return a new draw context for the 3D core.
*/
struct kgsl_context *
adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
uint32_t *flags)
{
struct adreno_context *drawctxt;
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return ERR_PTR(-ENOMEM);
ret = kgsl_context_init(dev_priv, &drawctxt->base);
if (ret != 0) {
kfree(drawctxt);
return ERR_PTR(ret);
}
drawctxt->bin_base_offset = 0;
drawctxt->timestamp = 0;
drawctxt->base.flags = *flags & (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
KGSL_CONTEXT_PER_CONTEXT_TS |
KGSL_CONTEXT_USER_GENERATED_TS |
KGSL_CONTEXT_NO_FAULT_TOLERANCE |
KGSL_CONTEXT_TYPE_MASK);
/* Always enable per-context timestamps */
drawctxt->base.flags |= KGSL_CONTEXT_PER_CONTEXT_TS;
mutex_init(&drawctxt->mutex);
init_waitqueue_head(&drawctxt->wq);
init_waitqueue_head(&drawctxt->waiting);
/*
* Set up the plist node for the dispatcher. For now all contexts have
* the same priority, but later the priority will be set at create time
* by the user
*/
plist_node_init(&drawctxt->pending, ADRENO_CONTEXT_DEFAULT_PRIORITY);
if (adreno_dev->gpudev->ctxt_create) {
ret = adreno_dev->gpudev->ctxt_create(adreno_dev, drawctxt);
if (ret)
goto err;
} else if ((drawctxt->base.flags & KGSL_CONTEXT_PREAMBLE) == 0 ||
(drawctxt->base.flags & KGSL_CONTEXT_NO_GMEM_ALLOC) == 0) {
KGSL_DEV_ERR_ONCE(device,
"legacy context switch not supported\n");
ret = -EINVAL;
goto err;
} else {
drawctxt->ops = &adreno_preamble_ctx_ops;
}
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
0);
/* copy back whatever flags we dediced were valid */
*flags = drawctxt->base.flags;
return &drawctxt->base;
err:
kgsl_context_detach(&drawctxt->base);
return ERR_PTR(ret);
}
/**
* adreno_drawctxt_detach(): detach a context from the GPU
* @context: Generic KGSL context container for the context
*
*/
void adreno_drawctxt_detach(struct kgsl_context *context)
{
struct kgsl_device *device;
struct adreno_device *adreno_dev;
struct adreno_context *drawctxt;
struct adreno_ringbuffer *rb;
int ret, count, i;
struct kgsl_cmdbatch *list[ADRENO_CONTEXT_CMDQUEUE_SIZE];
if (context == NULL)
return;
device = context->device;
adreno_dev = ADRENO_DEVICE(device);
drawctxt = ADRENO_CONTEXT(context);
rb = drawctxt->rb;
/* deactivate context */
mutex_lock(&device->mutex);
if (rb->drawctxt_active == drawctxt) {
if (adreno_dev->cur_rb == rb) {
if (!kgsl_active_count_get(device)) {
adreno_drawctxt_switch(adreno_dev, rb, NULL, 0);
kgsl_active_count_put(device);
} else
BUG();
} else
adreno_drawctxt_switch(adreno_dev, rb, NULL, 0);
}
mutex_unlock(&device->mutex);
spin_lock(&drawctxt->lock);
count = drawctxt_detach_cmdbatches(drawctxt, list);
spin_unlock(&drawctxt->lock);
for (i = 0; i < count; i++) {
/*
* If the context is deteached while we are waiting for
* the next command in GFT SKIP CMD, print the context
* detached status here.
*/
adreno_fault_skipcmd_detached(device, drawctxt, list[i]);
kgsl_cmdbatch_destroy(list[i]);
}
/*
* internal_timestamp is set in adreno_ringbuffer_addcmds,
* which holds the device mutex.
*/
mutex_lock(&device->mutex);
/*
* Wait for the last global timestamp to pass before continuing.
* The maxumum wait time is 30s, some large IB's can take longer
* than 10s and if hang happens then the time for the context's
* commands to retire will be greater than 10s. 30s should be sufficient
* time to wait for the commands even if a hang happens.
*/
ret = adreno_drawctxt_wait_rb(adreno_dev, context,
drawctxt->internal_timestamp, 30 * 1000);
/*
* If the wait for global fails due to timeout then nothing after this
* point is likely to work very well - BUG_ON() so we can take advantage
* of the debug tools to figure out what the h - e - double hockey
* sticks happened. If EAGAIN error is returned then recovery will kick
* in and there will be no more commands in the RB pipe from this
* context which is waht we are waiting for, so ignore -EAGAIN error
*/
BUG_ON(ret && ret != -EAGAIN);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
adreno_profile_process_results(adreno_dev);
mutex_unlock(&device->mutex);
/* wake threads waiting to submit commands from this context */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
}
/**
* adreno_drawctxt_create - create a new adreno draw context
* @dev_priv: the owner of the context
* @flags: flags for the context (passed from user space)
*
* Create and return a new draw context for the 3D core.
*/
struct kgsl_context *
adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
uint32_t *flags)
{
struct adreno_context *drawctxt;
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
int ret;
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return ERR_PTR(-ENOMEM);
ret = kgsl_context_init(dev_priv, &drawctxt->base);
if (ret != 0) {
kfree(drawctxt);
return ERR_PTR(ret);
}
drawctxt->bin_base_offset = 0;
drawctxt->timestamp = 0;
*flags &= (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
KGSL_CONTEXT_PER_CONTEXT_TS |
KGSL_CONTEXT_USER_GENERATED_TS |
KGSL_CONTEXT_NO_FAULT_TOLERANCE |
KGSL_CONTEXT_TYPE_MASK);
if (*flags & KGSL_CONTEXT_PREAMBLE)
drawctxt->flags |= CTXT_FLAGS_PREAMBLE;
if (*flags & KGSL_CONTEXT_NO_GMEM_ALLOC)
drawctxt->flags |= CTXT_FLAGS_NOGMEMALLOC;
if (*flags & KGSL_CONTEXT_PER_CONTEXT_TS)
drawctxt->flags |= CTXT_FLAGS_PER_CONTEXT_TS;
if (*flags & KGSL_CONTEXT_USER_GENERATED_TS) {
if (!(*flags & KGSL_CONTEXT_PER_CONTEXT_TS)) {
ret = -EINVAL;
goto err;
}
drawctxt->flags |= CTXT_FLAGS_USER_GENERATED_TS;
}
if (*flags & KGSL_CONTEXT_NO_FAULT_TOLERANCE)
drawctxt->flags |= CTXT_FLAGS_NO_FAULT_TOLERANCE;
drawctxt->type =
(*flags & KGSL_CONTEXT_TYPE_MASK) >> KGSL_CONTEXT_TYPE_SHIFT;
ret = adreno_dev->gpudev->ctxt_create(adreno_dev, drawctxt);
if (ret)
goto err;
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, ref_wait_ts),
KGSL_INIT_REFTIMESTAMP);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, ts_cmp_enable),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
0);
return &drawctxt->base;
err:
kgsl_context_put(&drawctxt->base);
return ERR_PTR(ret);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
if (context != NULL) {
/*
* if the context was not created with per context timestamp
* support, we must use the global timestamp since issueibcmds
* will be returning that one.
*/
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
}
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
total_sizedwords += 2; /* scratchpad ts for recovery */
if (context) {
total_sizedwords += 3; /* sop timestamp */
total_sizedwords += 4; /* eop timestamp */
total_sizedwords += 3; /* global timestamp without cache
* flush for non-zero context */
} else {
total_sizedwords += 4; /* global timestamp for recovery*/
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
/* always increment the global timestamp. once. */
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
/* scratchpad ts for recovery */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (adreno_is_a3xx(adreno_dev)) {
/*
* FLush HLSQ lazy updates to make sure there are no
* rsources pending for indirect loads after the timestamp
*/
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
if (context) {
/* start-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* end-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static void a2xx_drawctxt_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device = &adreno_dev->dev;
unsigned int cmds[5];
if (context == NULL) {
kgsl_mmu_setstate(&device->mmu, device->mmu.defaultpagetable,
adreno_dev->drawctxt_active->id);
return;
}
KGSL_CTXT_INFO(device, "context flags %08x\n", context->flags);
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
cmds[4] = context->id;
adreno_ringbuffer_issuecmds(device, context, KGSL_CMD_FLAGS_NONE,
cmds, 5);
kgsl_mmu_setstate(&device->mmu, context->pagetable, context->id);
#ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP
kgsl_cffdump_syncmem(NULL, &context->gpustate,
context->gpustate.gpuaddr, LCC_SHADOW_SIZE +
REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, false);
#endif
if (context->flags & CTXT_FLAGS_GMEM_RESTORE) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_PMODE,
context->context_gmem_shadow.gmem_restore, 3);
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE,
context->chicken_restore, 3);
}
context->flags &= ~CTXT_FLAGS_GMEM_RESTORE;
}
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, context->reg_restore, 3);
if (context->flags & CTXT_FLAGS_SHADER_RESTORE) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE,
context->shader_restore, 3);
}
}
if (adreno_is_a20x(adreno_dev)) {
cmds[0] = cp_type3_packet(CP_SET_BIN_BASE_OFFSET, 1);
cmds[1] = context->bin_base_offset;
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, cmds, 2);
}
}
