static uint32_t
kgsl_ringbuffer_addcmds(struct kgsl_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 9 : 0;
ringcmds = kgsl_ringbuffer_allocspace(rb, total_sizedwords);
if (flags & KGSL_CMD_FLAGS_PMODE) {
*ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1);
*ringcmds++ = 0;
}
memcpy(ringcmds, cmds, (sizedwords << 2));
ringcmds += sizedwords;
if (flags & KGSL_CMD_FLAGS_PMODE) {
*ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1);
*ringcmds++ = 1;
}
rb->timestamp++;
timestamp = rb->timestamp;
*ringcmds++ = pm4_type0_packet(REG_CP_TIMESTAMP, 1);
*ringcmds++ = rb->timestamp;
*ringcmds++ = pm4_type3_packet(PM4_EVENT_WRITE, 3);
*ringcmds++ = CACHE_FLUSH_TS;
*ringcmds++ =
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp));
*ringcmds++ = rb->timestamp;
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
*ringcmds++ = pm4_type3_packet(PM4_COND_EXEC, 4);
*ringcmds++ = (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2;
*ringcmds++ = (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2;
*ringcmds++ = rb->timestamp;
*ringcmds++ = 4;
*ringcmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1);
*ringcmds++ = 0x00000000;
*ringcmds++ = pm4_type3_packet(PM4_INTERRUPT, 1);
*ringcmds++ = CP_INT_CNTL__RB_INT_MASK;
}
static void a2xx_ctxt_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device = &adreno_dev->dev;
unsigned int cmds[5];
if (context == NULL) {
/* No context - set the default apgetable and thats it */
kgsl_mmu_setstate(device, device->mmu.defaultpagetable);
return;
}
KGSL_CTXT_INFO(device, "context flags %08x\n", context->flags);
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(current_context);
cmds[4] = (unsigned int) context;
adreno_ringbuffer_issuecmds(device, 0, cmds, 5);
kgsl_mmu_setstate(device, context->pagetable);
#ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP
kgsl_cffdump_syncmem(NULL, &context->gpustate,
context->gpustate.gpuaddr, LCC_SHADOW_SIZE +
REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, false);
#endif
/* restore gmem.
* (note: changes shader. shader must not already be restored.)
*/
if (context->flags & CTXT_FLAGS_GMEM_RESTORE) {
adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE,
context->context_gmem_shadow.gmem_restore, 3);
/* Restore TP0_CHICKEN */
adreno_ringbuffer_issuecmds(device, 0,
context->chicken_restore, 3);
context->flags &= ~CTXT_FLAGS_GMEM_RESTORE;
}
/* restore registers and constants. */
adreno_ringbuffer_issuecmds(device, 0,
context->reg_restore, 3);
/* restore shader instructions & partitioning. */
if (context->flags & CTXT_FLAGS_SHADER_RESTORE) {
adreno_ringbuffer_issuecmds(device, 0,
context->shader_restore, 3);
}
if (adreno_is_a20x(adreno_dev)) {
cmds[0] = cp_type3_packet(CP_SET_BIN_BASE_OFFSET, 1);
cmds[1] = context->bin_base_offset;
adreno_ringbuffer_issuecmds(device, 0, cmds, 2);
}
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
if (adreno_is_a2xx(adreno_dev))
total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */
if (adreno_is_a20x(adreno_dev))
total_sizedwords += 2; /* CACHE_FLUSH */
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
/* GPU may hang during space allocation, if thats the case the current
* context may have hung the GPU */
if (context && context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp;
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* HW Workaround for MMU Page fault
* due to memory getting free early before
* GPU completes it.
*/
if (adreno_is_a2xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (adreno_is_a20x(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
{
int status;
/*cp_rb_cntl_u cp_rb_cntl; */
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int *cmds, rb_cntl;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
uint cmds_gpu;
if (rb->flags & KGSL_FLAGS_STARTED)
return 0;
if (init_ram) {
rb->timestamp = 0;
GSL_RB_INIT_TIMESTAMP(rb);
}
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
/* setup WPTR delay */
adreno_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */);
/*setup REG_CP_RB_CNTL */
adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
/*
* The size of the ringbuffer in the hardware is the log2
* representation of the size in quadwords (sizedwords / 2)
*/
cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
/*
* Specify the quadwords to read before updating mem RPTR.
* Like above, pass the log2 representation of the blocksize
* in quadwords.
*/
cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; /* WPTR polling */
/* mem RPTR writebacks */
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
/* explicitly clear all cp interrupts */
adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
/* setup scratch/timestamp */
adreno_regwrite(device, REG_SCRATCH_ADDR,
device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp));
adreno_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
/* update the eoptimestamp field with the last retired timestamp */
kgsl_sharedmem_writel(&device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp),
rb->timestamp);
/* load the CP ucode */
status = adreno_ringbuffer_load_pm4_ucode(device);
if (status != 0)
return status;
/* load the prefetch parser ucode */
status = adreno_ringbuffer_load_pfp_ucode(device);
if (status != 0)
return status;
adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804);
rb->rptr = 0;
rb->wptr = 0;
/* clear ME_HALT to start micro engine */
adreno_regwrite(device, REG_CP_ME_CNTL, 0);
/* ME_INIT */
cmds = adreno_ringbuffer_allocspace(rb, 19);
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-19);
GSL_RB_WRITE(cmds, cmds_gpu, CP_HDR_ME_INIT);
/* All fields present (bits 9:0) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x000003ff);
/* Disable/Enable Real-Time Stream processing (present but ignored) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Enable (2D <-> 3D) implicit synchronization (present but ignored) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE));
/* Instruction memory size: */
GSL_RB_WRITE(cmds, cmds_gpu,
(adreno_encode_istore_size(adreno_dev)
| adreno_dev->pix_shader_start));
/* Maximum Contexts */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000001);
/* Write Confirm Interval and The CP will wait the
* wait_interval * 16 clocks between polling */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* NQ and External Memory Swap */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Protected mode error checking */
GSL_RB_WRITE(cmds, cmds_gpu, GSL_RB_PROTECTED_MODE_CONTROL);
/* Disable header dumping and Header dump address */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Header dump size */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
adreno_ringbuffer_submit(rb);
/* idle device to validate ME INIT */
status = adreno_idle(device);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
return status;
}
/* functions */
void kgsl_cp_intrcallback(struct kgsl_device *device)
{
unsigned int status = 0, num_reads = 0, master_status = 0;
struct kgsl_yamato_device *yamato_device = KGSL_YAMATO_DEVICE(device);
struct kgsl_ringbuffer *rb = &yamato_device->ringbuffer;
kgsl_yamato_regread_isr(device, REG_MASTER_INT_SIGNAL, &master_status);
while (!status && (num_reads < VALID_STATUS_COUNT_MAX) &&
(master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) {
kgsl_yamato_regread_isr(device, REG_CP_INT_STATUS, &status);
kgsl_yamato_regread_isr(device, REG_MASTER_INT_SIGNAL,
&master_status);
num_reads++;
}
if (num_reads > 1)
KGSL_DRV_WARN(device,
"Looped %d times to read REG_CP_INT_STATUS\n",
num_reads);
if (!status) {
if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) {
/* This indicates that we could not read CP_INT_STAT.
* As a precaution just wake up processes so
* they can check their timestamps. Since, we
* did not ack any interrupts this interrupt will
* be generated again */
KGSL_DRV_WARN(device, "Unable to read CP_INT_STATUS\n");
wake_up_interruptible_all(&device->wait_queue);
} else
KGSL_DRV_WARN(device, "Spurious interrput detected\n");
return;
}
if (status & CP_INT_CNTL__RB_INT_MASK) {
/* signal intr completion event */
unsigned int enableflag = 0;
kgsl_sharedmem_writel(&rb->device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable),
enableflag);
wmb();
KGSL_CMD_WARN(rb->device, "ringbuffer rb interrupt\n");
}
if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) {
KGSL_CMD_CRIT(rb->device,
"ringbuffer TO packet in IB interrupt\n");
kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0);
}
if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) {
KGSL_CMD_CRIT(rb->device,
"ringbuffer opcode error interrupt\n");
kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0);
}
if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) {
KGSL_CMD_CRIT(rb->device,
"ringbuffer protected mode error interrupt\n");
kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0);
}
if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) {
KGSL_CMD_CRIT(rb->device,
"ringbuffer reserved bit error interrupt\n");
kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0);
}
if (status & CP_INT_CNTL__IB_ERROR_MASK) {
KGSL_CMD_CRIT(rb->device,
"ringbuffer IB error interrupt\n");
kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0);
}
if (status & CP_INT_CNTL__SW_INT_MASK)
KGSL_CMD_INFO(rb->device, "ringbuffer software interrupt\n");
if (status & CP_INT_CNTL__IB2_INT_MASK)
KGSL_CMD_INFO(rb->device, "ringbuffer ib2 interrupt\n");
if (status & (~GSL_CP_INT_MASK))
KGSL_CMD_WARN(rb->device,
"bad bits in REG_CP_INT_STATUS %08x\n", status);
/* only ack bits we understand */
status &= GSL_CP_INT_MASK;
kgsl_yamato_regwrite_isr(device, REG_CP_INT_ACK, status);
if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) {
KGSL_CMD_WARN(rb->device, "ringbuffer ib1/rb interrupt\n");
wake_up_interruptible_all(&device->wait_queue);
atomic_notifier_call_chain(&(device->ts_notifier_list),
KGSL_DEVICE_YAMATO,
NULL);
}
}
static uint32_t
kgsl_ringbuffer_addcmds(struct kgsl_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
ringcmds = kgsl_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* start-of-pipeline and end-of-pipeline timestamps */
GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
pm4_type3_packet(PM4_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
pm4_type3_packet(PM4_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static void a2xx_cp_intrcallback(struct kgsl_device *device)
{
unsigned int status = 0, num_reads = 0, master_status = 0;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
int i;
adreno_regread(device, REG_MASTER_INT_SIGNAL, &master_status);
while (!status && (num_reads < VALID_STATUS_COUNT_MAX) &&
(master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) {
adreno_regread(device, REG_CP_INT_STATUS, &status);
adreno_regread(device, REG_MASTER_INT_SIGNAL,
&master_status);
num_reads++;
}
if (num_reads > 1)
KGSL_DRV_WARN(device,
"Looped %d times to read REG_CP_INT_STATUS\n",
num_reads);
if (!status) {
if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) {
/* This indicates that we could not read CP_INT_STAT.
* As a precaution just wake up processes so
* they can check their timestamps. Since, we
* did not ack any interrupts this interrupt will
* be generated again */
KGSL_DRV_WARN(device, "Unable to read CP_INT_STATUS\n");
wake_up_interruptible_all(&device->wait_queue);
} else
KGSL_DRV_WARN(device, "Spurious interrput detected\n");
return;
}
if (status & CP_INT_CNTL__RB_INT_MASK) {
/* signal intr completion event */
unsigned int enableflag = 0;
kgsl_sharedmem_writel(&rb->device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable),
enableflag);
wmb();
KGSL_CMD_WARN(rb->device, "ringbuffer rb interrupt\n");
}
for (i = 0; i < ARRAY_SIZE(kgsl_cp_error_irqs); i++) {
if (status & kgsl_cp_error_irqs[i].mask) {
KGSL_CMD_CRIT(rb->device, "%s\n",
kgsl_cp_error_irqs[i].message);
/*
* on fatal errors, turn off the interrupts to
* avoid storming. This has the side effect of
* forcing a PM dump when the timestamp times out
*/
kgsl_pwrctrl_irq(rb->device, KGSL_PWRFLAGS_OFF);
}
}
/* only ack bits we understand */
status &= CP_INT_MASK;
adreno_regwrite(device, REG_CP_INT_ACK, status);
if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) {
KGSL_CMD_WARN(rb->device, "ringbuffer ib1/rb interrupt\n");
queue_work(device->work_queue, &device->ts_expired_ws);
wake_up_interruptible_all(&device->wait_queue);
atomic_notifier_call_chain(&(device->ts_notifier_list),
device->id,
NULL);
}
}
/* switch drawing contexts */
void
adreno_drawctxt_switch(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt,
unsigned int flags)
{
struct adreno_context *active_ctxt =
adreno_dev->drawctxt_active;
struct kgsl_device *device = &adreno_dev->dev;
unsigned int cmds[5];
if (drawctxt) {
if (flags & KGSL_CONTEXT_SAVE_GMEM)
/* Set the flag in context so that the save is done
* when this context is switched out. */
drawctxt->flags |= CTXT_FLAGS_GMEM_SAVE;
else
/* Remove GMEM saving flag from the context */
drawctxt->flags &= ~CTXT_FLAGS_GMEM_SAVE;
}
/* already current? */
if (active_ctxt == drawctxt)
return;
KGSL_CTXT_INFO(device, "from %p to %p flags %d\n",
adreno_dev->drawctxt_active, drawctxt, flags);
/* save old context*/
if (active_ctxt && active_ctxt->flags & CTXT_FLAGS_GPU_HANG)
KGSL_CTXT_WARN(device,
"Current active context has caused gpu hang\n");
if (active_ctxt != NULL) {
KGSL_CTXT_INFO(device,
"active_ctxt flags %08x\n", active_ctxt->flags);
/* save registers and constants. */
adreno_ringbuffer_issuecmds(device, 0,
active_ctxt->reg_save, 3);
if (active_ctxt->flags & CTXT_FLAGS_SHADER_SAVE) {
/* save shader partitioning and instructions. */
adreno_ringbuffer_issuecmds(device,
KGSL_CMD_FLAGS_PMODE,
active_ctxt->shader_save, 3);
/* fixup shader partitioning parameter for
* SET_SHADER_BASES.
*/
adreno_ringbuffer_issuecmds(device, 0,
active_ctxt->shader_fixup, 3);
active_ctxt->flags |= CTXT_FLAGS_SHADER_RESTORE;
}
if (active_ctxt->flags & CTXT_FLAGS_GMEM_SAVE
&& active_ctxt->flags & CTXT_FLAGS_GMEM_SHADOW) {
/* save gmem.
* (note: changes shader. shader must already be saved.)
*/
adreno_ringbuffer_issuecmds(device,
KGSL_CMD_FLAGS_PMODE,
active_ctxt->context_gmem_shadow.gmem_save, 3);
/* Restore TP0_CHICKEN */
adreno_ringbuffer_issuecmds(device, 0,
active_ctxt->chicken_restore, 3);
active_ctxt->flags |= CTXT_FLAGS_GMEM_RESTORE;
}
}
adreno_dev->drawctxt_active = drawctxt;
/* restore new context */
if (drawctxt != NULL) {
KGSL_CTXT_INFO(device,
"drawctxt flags %08x\n", drawctxt->flags);
cmds[0] = pm4_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = pm4_type3_packet(PM4_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(current_context);
cmds[4] = (unsigned int)adreno_dev->drawctxt_active;
adreno_ringbuffer_issuecmds(device, 0, cmds, 5);
kgsl_mmu_setstate(device, drawctxt->pagetable);
#ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP
kgsl_cffdump_syncmem(NULL, &drawctxt->gpustate,
drawctxt->gpustate.gpuaddr, LCC_SHADOW_SIZE +
REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE,
false);
#endif
/* restore gmem.
* (note: changes shader. shader must not already be restored.)
*/
if (drawctxt->flags & CTXT_FLAGS_GMEM_RESTORE) {
adreno_ringbuffer_issuecmds(device,
KGSL_CMD_FLAGS_PMODE,
drawctxt->context_gmem_shadow.gmem_restore, 3);
/* Restore TP0_CHICKEN */
adreno_ringbuffer_issuecmds(device, 0,
drawctxt->chicken_restore, 3);
drawctxt->flags &= ~CTXT_FLAGS_GMEM_RESTORE;
}
/* restore registers and constants. */
adreno_ringbuffer_issuecmds(device, 0,
drawctxt->reg_restore, 3);
/* restore shader instructions & partitioning. */
if (drawctxt->flags & CTXT_FLAGS_SHADER_RESTORE) {
adreno_ringbuffer_issuecmds(device, 0,
drawctxt->shader_restore, 3);
}
cmds[0] = pm4_type3_packet(PM4_SET_BIN_BASE_OFFSET, 1);
cmds[1] = drawctxt->bin_base_offset;
if (!adreno_is_a220(adreno_dev))
adreno_ringbuffer_issuecmds(device, 0, cmds, 2);
} else
kgsl_mmu_setstate(device, device->mmu.defaultpagetable);
}
int kgsl_ringbuffer_start(struct kgsl_ringbuffer *rb, unsigned int init_ram)
{
int status;
/*cp_rb_cntl_u cp_rb_cntl; */
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int *cmds, rb_cntl;
struct kgsl_device *device = rb->device;
uint cmds_gpu;
KGSL_CMD_VDBG("enter (rb=%p)\n", rb);
if (rb->flags & KGSL_FLAGS_STARTED) {
KGSL_CMD_VDBG("already started return %d\n", 0);
return 0;
}
if (init_ram) {
rb->timestamp = 0;
GSL_RB_INIT_TIMESTAMP(rb);
}
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
/* setup WPTR delay */
kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */);
/*setup REG_CP_RB_CNTL */
kgsl_yamato_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
/* size of ringbuffer */
cp_rb_cntl.f.rb_bufsz =
kgsl_ringbuffer_sizelog2quadwords(rb->sizedwords);
/* quadwords to read before updating mem RPTR */
cp_rb_cntl.f.rb_blksz = rb->blksizequadwords;
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; /* WPTR polling */
/* mem RPTR writebacks */
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
kgsl_yamato_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
kgsl_yamato_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
kgsl_yamato_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
/* explicitly clear all cp interrupts */
kgsl_yamato_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
/* setup scratch/timestamp */
kgsl_yamato_regwrite(device, REG_SCRATCH_ADDR,
device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp));
kgsl_yamato_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
/* load the CP ucode */
status = kgsl_ringbuffer_load_pm4_ucode(device);
if (status != 0) {
KGSL_DRV_ERR("kgsl_ringbuffer_load_pm4_ucode failed %d\n",
status);
return status;
}
/* load the prefetch parser ucode */
status = kgsl_ringbuffer_load_pfp_ucode(device);
if (status != 0) {
KGSL_DRV_ERR("kgsl_ringbuffer_load_pfp_ucode failed %d\n",
status);
return status;
}
kgsl_yamato_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804);
rb->rptr = 0;
rb->wptr = 0;
/* clear ME_HALT to start micro engine */
kgsl_yamato_regwrite(device, REG_CP_ME_CNTL, 0);
/* ME_INIT */
cmds = kgsl_ringbuffer_allocspace(rb, 19);
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-19);
GSL_RB_WRITE(cmds, cmds_gpu, PM4_HDR_ME_INIT);
/* All fields present (bits 9:0) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x000003ff);
/* Disable/Enable Real-Time Stream processing (present but ignored) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Enable (2D <-> 3D) implicit synchronization (present but ignored) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL));
GSL_RB_WRITE(cmds, cmds_gpu,
GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE));
/* Vertex and Pixel Shader Start Addresses in instructions
* (3 DWORDS per instruction) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x80000180);
/* Maximum Contexts */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000001);
/* Write Confirm Interval and The CP will wait the
* wait_interval * 16 clocks between polling */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* NQ and External Memory Swap */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Protected mode error checking */
GSL_RB_WRITE(cmds, cmds_gpu, GSL_RB_PROTECTED_MODE_CONTROL);
/* Disable header dumping and Header dump address */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Header dump size */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
kgsl_ringbuffer_submit(rb);
/* idle device to validate ME INIT */
status = kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT);
KGSL_CMD_DBG("enabling CP interrupts: mask %08lx\n", GSL_CP_INT_MASK);
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, GSL_CP_INT_MASK);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
KGSL_CMD_VDBG("return %d\n", status);
return status;
}
void kgsl_cp_intrcallback(struct kgsl_device *device)
{
unsigned int status = 0, num_reads = 0, master_status = 0;
struct kgsl_yamato_device *yamato_device = (struct kgsl_yamato_device *)
device;
struct kgsl_ringbuffer *rb = &device->ringbuffer;
KGSL_CMD_VDBG("enter (device=%p)\n", device);
kgsl_yamato_regread(device, REG_MASTER_INT_SIGNAL, &master_status);
while (!status && (num_reads < VALID_STATUS_COUNT_MAX) &&
(master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) {
kgsl_yamato_regread(device, REG_CP_INT_STATUS, &status);
kgsl_yamato_regread(device, REG_MASTER_INT_SIGNAL,
&master_status);
num_reads++;
}
if (num_reads > 1)
KGSL_DRV_WARN("Looped %d times to read REG_CP_INT_STATUS\n",
num_reads);
if (!status) {
if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) {
KGSL_DRV_WARN("Unable to read CP_INT_STATUS\n");
wake_up_interruptible_all(&yamato_device->ib1_wq);
} else
KGSL_DRV_WARN("Spurious interrput detected\n");
return;
}
if (status & CP_INT_CNTL__RB_INT_MASK) {
unsigned int enableflag = 0;
kgsl_sharedmem_writel(&rb->device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable),
enableflag);
wmb();
KGSL_CMD_WARN("ringbuffer rb interrupt\n");
}
if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) {
KGSL_CMD_FATAL("ringbuffer TO packet in IB interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer opcode error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer protected mode error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer reserved bit error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__IB_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer IB error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__SW_INT_MASK)
KGSL_CMD_DBG("ringbuffer software interrupt\n");
if (status & CP_INT_CNTL__IB2_INT_MASK)
KGSL_CMD_DBG("ringbuffer ib2 interrupt\n");
if (status & (~GSL_CP_INT_MASK))
KGSL_CMD_DBG("bad bits in REG_CP_INT_STATUS %08x\n", status);
status &= GSL_CP_INT_MASK;
kgsl_yamato_regwrite(device, REG_CP_INT_ACK, status);
if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) {
KGSL_CMD_WARN("ringbuffer ib1/rb interrupt\n");
wake_up_interruptible_all(&yamato_device->ib1_wq);
atomic_notifier_call_chain(&(device->ts_notifier_list),
KGSL_DEVICE_YAMATO,
NULL);
}
KGSL_CMD_VDBG("return\n");
}
static int kgsl_ringbuffer_start(struct kgsl_ringbuffer *rb)
{
int status;
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int *cmds, rb_cntl;
struct kgsl_device *device = rb->device;
KGSL_CMD_VDBG("enter (rb=%p)\n", rb);
if (rb->flags & KGSL_FLAGS_STARTED) {
KGSL_CMD_VDBG("return %d\n", 0);
return 0;
}
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 );
kgsl_yamato_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
cp_rb_cntl.f.rb_bufsz =
kgsl_ringbuffer_sizelog2quadwords(rb->sizedwords);
cp_rb_cntl.f.rb_blksz = rb->blksizequadwords;
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
kgsl_yamato_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
kgsl_yamato_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
kgsl_yamato_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
kgsl_yamato_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
kgsl_yamato_regwrite(device, REG_SCRATCH_ADDR,
device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp));
kgsl_yamato_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
status = kgsl_ringbuffer_load_pm4_ucode(device);
if (status != 0) {
KGSL_DRV_ERR("kgsl_ringbuffer_load_pm4_ucode failed %d\n",
status);
return status;
}
status = kgsl_ringbuffer_load_pfp_ucode(device);
if (status != 0) {
KGSL_DRV_ERR("kgsl_ringbuffer_load_pm4_ucode failed %d\n",
status);
return status;
}
kgsl_yamato_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804);
rb->rptr = 0;
rb->wptr = 0;
rb->timestamp = 0;
GSL_RB_INIT_TIMESTAMP(rb);
INIT_LIST_HEAD(&rb->memqueue);
kgsl_yamato_regwrite(device, REG_CP_ME_CNTL, 0);
cmds = kgsl_ringbuffer_allocspace(rb, 19);
GSL_RB_WRITE(cmds, PM4_HDR_ME_INIT);
GSL_RB_WRITE(cmds, 0x000003ff);
GSL_RB_WRITE(cmds, 0x00000000);
GSL_RB_WRITE(cmds, 0x00000000);
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO));
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET));
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX));
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL));
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL));
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE));
GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL));
GSL_RB_WRITE(cmds,
GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE));
GSL_RB_WRITE(cmds, 0x80000180);
GSL_RB_WRITE(cmds, 0x00000001);
GSL_RB_WRITE(cmds, 0x00000000);
GSL_RB_WRITE(cmds, 0x00000000);
GSL_RB_WRITE(cmds, GSL_RB_PROTECTED_MODE_CONTROL);
GSL_RB_WRITE(cmds, 0x00000000);
GSL_RB_WRITE(cmds, 0x00000000);
kgsl_ringbuffer_submit(rb);
status = kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT);
KGSL_CMD_DBG("enabling CP interrupts: mask %08lx\n", GSL_CP_INT_MASK);
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, GSL_CP_INT_MASK);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
KGSL_CMD_VDBG("return %d\n", status);
return status;
}
static uint32_t
kgsl_ringbuffer_addcmds(struct kgsl_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 9 : 0;
ringcmds = kgsl_ringbuffer_allocspace(rb, total_sizedwords);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
*ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1);
*ringcmds++ = 0;
}
memcpy(ringcmds, cmds, (sizedwords << 2));
ringcmds += sizedwords;
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
*ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1);
*ringcmds++ = 1;
}
rb->timestamp++;
timestamp = rb->timestamp;
/* start-of-pipeline and end-of-pipeline timestamps */
*ringcmds++ = pm4_type0_packet(REG_CP_TIMESTAMP, 1);
*ringcmds++ = rb->timestamp;
*ringcmds++ = pm4_type3_packet(PM4_EVENT_WRITE, 3);
*ringcmds++ = CACHE_FLUSH_TS;
*ringcmds++ =
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp));
*ringcmds++ = rb->timestamp;
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
*ringcmds++ = pm4_type3_packet(PM4_COND_EXEC, 4);
*ringcmds++ = (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2;
*ringcmds++ = (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2;
*ringcmds++ = rb->timestamp;
/* # of conditional command DWORDs */
*ringcmds++ = 4;
*ringcmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1);
*ringcmds++ = 0x00000000;
*ringcmds++ = pm4_type3_packet(PM4_INTERRUPT, 1);
*ringcmds++ = CP_INT_CNTL__RB_INT_MASK;
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* start-of-pipeline and end-of-pipeline timestamps */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (adreno_is_a3xx(adreno_dev)) {
/*
* FLush HLSQ lazy updates to make sure there are no
* rsources pending for indirect loads after the timestamp
*/
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
{
int status;
/*cp_rb_cntl_u cp_rb_cntl; */
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int rb_cntl;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (rb->flags & KGSL_FLAGS_STARTED)
return 0;
if (init_ram) {
rb->timestamp = 0;
GSL_RB_INIT_TIMESTAMP(rb);
}
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
if (adreno_is_a2xx(adreno_dev)) {
adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
/* setup WPTR delay */
adreno_regwrite(device, REG_CP_RB_WPTR_DELAY,
0 /*0x70000010 */);
}
/*setup REG_CP_RB_CNTL */
adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
/*
* The size of the ringbuffer in the hardware is the log2
* representation of the size in quadwords (sizedwords / 2)
*/
cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
/*
* Specify the quadwords to read before updating mem RPTR.
* Like above, pass the log2 representation of the blocksize
* in quadwords.
*/
cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
if (adreno_is_a2xx(adreno_dev)) {
/* WPTR polling */
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
}
/* mem RPTR writebacks */
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
if (adreno_is_a3xx(adreno_dev)) {
/* enable access protection to privileged registers */
adreno_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007);
/* RBBM registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_0, 0x63000040);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_1, 0x62000080);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_2, 0x600000CC);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_3, 0x60000108);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_4, 0x64000140);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_5, 0x66000400);
/* CP registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_6, 0x65000700);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_7, 0x610007D8);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_8, 0x620007E0);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_9, 0x61001178);
adreno_regwrite(device, A3XX_CP_PROTECT_REG_A, 0x64001180);
/* RB registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_B, 0x60003300);
/* VBIF registers */
adreno_regwrite(device, A3XX_CP_PROTECT_REG_C, 0x6B00C000);
}
if (adreno_is_a2xx(adreno_dev)) {
/* explicitly clear all cp interrupts */
adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
}
/* setup scratch/timestamp */
adreno_regwrite(device, REG_SCRATCH_ADDR,
device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp));
adreno_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
/* load the CP ucode */
status = adreno_ringbuffer_load_pm4_ucode(device);
if (status != 0)
return status;
/* load the prefetch parser ucode */
status = adreno_ringbuffer_load_pfp_ucode(device);
if (status != 0)
return status;
adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804);
rb->rptr = 0;
rb->wptr = 0;
/* clear ME_HALT to start micro engine */
adreno_regwrite(device, REG_CP_ME_CNTL, 0);
/* ME init is GPU specific, so jump into the sub-function */
adreno_dev->gpudev->rb_init(adreno_dev, rb);
/* idle device to validate ME INIT */
status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
return status;
}
/* functions */
void kgsl_cp_intrcallback(struct kgsl_device *device)
{
unsigned int status = 0;
struct kgsl_ringbuffer *rb = &device->ringbuffer;
KGSL_CMD_VDBG("enter (device=%p)\n", device);
kgsl_yamato_regread(device, REG_CP_INT_STATUS, &status);
if (status & CP_INT_CNTL__RB_INT_MASK) {
/* signal intr completion event */
int init_reftimestamp = 0x7fffffff;
int enableflag = 0;
kgsl_sharedmem_write(&rb->device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable),
&enableflag, 4);
kgsl_sharedmem_write(&rb->device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts),
&init_reftimestamp, 4);
KGSL_CMD_WARN("ringbuffer rb interrupt\n");
}
if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) {
KGSL_CMD_WARN("ringbuffer ib1/rb interrupt\n");
wake_up_interruptible_all(&device->ib1_wq);
}
if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) {
KGSL_CMD_FATAL("ringbuffer TO packet in IB interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer opcode error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer protected mode error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer reserved bit error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__IB_ERROR_MASK) {
KGSL_CMD_FATAL("ringbuffer IB error interrupt\n");
kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0);
kgsl_ringbuffer_dump(rb);
}
if (status & CP_INT_CNTL__SW_INT_MASK)
KGSL_CMD_DBG("ringbuffer software interrupt\n");
if (status & CP_INT_CNTL__IB2_INT_MASK)
KGSL_CMD_DBG("ringbuffer ib2 interrupt\n");
if (status & (~GSL_CP_INT_MASK))
KGSL_CMD_DBG("bad bits in REG_CP_INT_STATUS %08x\n", status);
/* only ack bits we understand */
status &= GSL_CP_INT_MASK;
kgsl_yamato_regwrite(device, REG_CP_INT_ACK, status);
KGSL_CMD_VDBG("return\n");
}
