static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
if (context != NULL) {
/*
* if the context was not created with per context timestamp
* support, we must use the global timestamp since issueibcmds
* will be returning that one.
*/
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
}
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
total_sizedwords += 2; /* scratchpad ts for recovery */
if (context) {
total_sizedwords += 3; /* sop timestamp */
total_sizedwords += 4; /* eop timestamp */
total_sizedwords += 3; /* global timestamp without cache
* flush for non-zero context */
} else {
total_sizedwords += 4; /* global timestamp for recovery*/
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
/* always increment the global timestamp. once. */
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
/* scratchpad ts for recovery */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (adreno_is_a3xx(adreno_dev)) {
/*
* FLush HLSQ lazy updates to make sure there are no
* rsources pending for indirect loads after the timestamp
*/
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
if (context) {
/* start-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* end-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
/**
* _ringbuffer_bootstrap_ucode() - Bootstrap GPU Ucode
* @rb: Pointer to adreno ringbuffer
* @load_jt: If non zero only load Jump tables
*
* Bootstrap ucode for GPU
* load_jt == 0, bootstrap full microcode
* load_jt == 1, bootstrap jump tables of microcode
*
* For example a bootstrap packet would like below
* Setup a type3 bootstrap packet
* PFP size to bootstrap
* PFP addr to write the PFP data
* PM4 size to bootstrap
* PM4 addr to write the PM4 data
* PFP dwords from microcode to bootstrap
* PM4 size dwords from microcode to bootstrap
*/
static int _ringbuffer_bootstrap_ucode(struct adreno_ringbuffer *rb,
unsigned int load_jt)
{
unsigned int *cmds, bootstrap_size;
int i = 0;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int pm4_size, pm4_idx, pm4_addr, pfp_size, pfp_idx, pfp_addr;
/* Only bootstrap jump tables of ucode */
if (load_jt) {
pm4_idx = adreno_dev->gpucore->pm4_jt_idx;
pm4_addr = adreno_dev->gpucore->pm4_jt_addr;
pfp_idx = adreno_dev->gpucore->pfp_jt_idx;
pfp_addr = adreno_dev->gpucore->pfp_jt_addr;
} else {
/* Bootstrap full ucode */
pm4_idx = 1;
pm4_addr = 0;
pfp_idx = 1;
pfp_addr = 0;
}
pm4_size = (adreno_dev->pm4_fw_size - pm4_idx);
pfp_size = (adreno_dev->pfp_fw_size - pfp_idx);
/*
* Overwrite the first entry in the jump table with the special
* bootstrap opcode
*/
if (adreno_is_a4xx(adreno_dev)) {
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR,
0x400);
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA,
0x6f0009);
bootstrap_size = (pm4_size + pfp_size + 5 + 6);
} else {
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR,
0x200);
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA,
0x6f0005);
bootstrap_size = (pm4_size + pfp_size + 5);
}
/* clear ME_HALT to start micro engine */
adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0);
cmds = adreno_ringbuffer_allocspace(rb, bootstrap_size);
if (IS_ERR(cmds))
return PTR_ERR(cmds);
if (cmds == NULL)
return -ENOSPC;
/* Construct the packet that bootsraps the ucode */
*cmds++ = cp_type3_packet(CP_BOOTSTRAP_UCODE, (bootstrap_size - 1));
*cmds++ = pfp_size;
*cmds++ = pfp_addr;
*cmds++ = pm4_size;
*cmds++ = pm4_addr;
/**
* Theory of operation:
*
* In A4x, we cannot have the PFP executing instructions while its instruction
* RAM is loading. We load the PFP's instruction RAM using type-0 writes
* from the ME.
*
* To make sure the PFP is not fetching instructions at the same time,
* we put it in a one-instruction loop:
* mvc (ME), (ringbuffer)
* which executes repeatedly until all of the data has been moved from
* the ring buffer to the ME.
*/
if (adreno_is_a4xx(adreno_dev)) {
for (i = pm4_idx; i < adreno_dev->pm4_fw_size; i++)
*cmds++ = adreno_dev->pm4_fw[i];
for (i = pfp_idx; i < adreno_dev->pfp_fw_size; i++)
*cmds++ = adreno_dev->pfp_fw[i];
*cmds++ = cp_type3_packet(CP_REG_RMW, 3);
*cmds++ = 0x20000000 + A4XX_CP_RB_WPTR;
*cmds++ = 0xffffffff;
*cmds++ = 0x00000002;
*cmds++ = cp_type3_packet(CP_INTERRUPT, 1);
*cmds++ = 0;
rb->wptr = rb->wptr - 2;
adreno_ringbuffer_submit(rb, NULL);
rb->wptr = rb->wptr + 2;
} else {
for (i = pfp_idx; i < adreno_dev->pfp_fw_size; i++)
*cmds++ = adreno_dev->pfp_fw[i];
for (i = pm4_idx; i < adreno_dev->pm4_fw_size; i++)
*cmds++ = adreno_dev->pm4_fw[i];
adreno_ringbuffer_submit(rb, NULL);
}
/* idle device to validate bootstrap */
return adreno_spin_idle(device);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
/*
* if the context was not created with per context timestamp
* support, we must use the global timestamp since issueibcmds
* will be returning that one.
*/
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
if (adreno_is_a2xx(adreno_dev))
total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */
total_sizedwords += 2; /* scratchpad ts for recovery */
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
total_sizedwords += 3; /* sop timestamp */
total_sizedwords += 4; /* eop timestamp */
total_sizedwords += 3; /* global timestamp without cache
* flush for non-zero context */
} else {
total_sizedwords += 4; /* global timestamp for recovery*/
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
/* GPU may hang during space allocation, if thats the case the current
* context may have hung the GPU */
if (context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp[context_id];
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
/* always increment the global timestamp. once. */
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
/* HW Workaround for MMU Page fault
* due to memory getting free early before
* GPU completes it.
*/
if (adreno_is_a2xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
/* scratchpad ts for recovery */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
/* start-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* end-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
if (adreno_is_a2xx(adreno_dev))
total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */
if (adreno_is_a20x(adreno_dev))
total_sizedwords += 2; /* CACHE_FLUSH */
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
/* GPU may hang during space allocation, if thats the case the current
* context may have hung the GPU */
if (context && context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp;
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* HW Workaround for MMU Page fault
* due to memory getting free early before
* GPU completes it.
*/
if (adreno_is_a2xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (adreno_is_a20x(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
{
int status;
/*cp_rb_cntl_u cp_rb_cntl; */
union reg_cp_rb_cntl cp_rb_cntl;
unsigned int *cmds, rb_cntl;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
uint cmds_gpu;
if (rb->flags & KGSL_FLAGS_STARTED)
return 0;
if (init_ram) {
rb->timestamp = 0;
GSL_RB_INIT_TIMESTAMP(rb);
}
kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
sizeof(struct kgsl_rbmemptrs));
kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
(rb->sizedwords << 2));
adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
(rb->memptrs_desc.gpuaddr
+ GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
/* setup WPTR delay */
adreno_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */);
/*setup REG_CP_RB_CNTL */
adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
cp_rb_cntl.val = rb_cntl;
/*
* The size of the ringbuffer in the hardware is the log2
* representation of the size in quadwords (sizedwords / 2)
*/
cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
/*
* Specify the quadwords to read before updating mem RPTR.
* Like above, pass the log2 representation of the blocksize
* in quadwords.
*/
cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; /* WPTR polling */
/* mem RPTR writebacks */
cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
rb->memptrs_desc.gpuaddr +
GSL_RB_MEMPTRS_RPTR_OFFSET);
/* explicitly clear all cp interrupts */
adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
/* setup scratch/timestamp */
adreno_regwrite(device, REG_SCRATCH_ADDR,
device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp));
adreno_regwrite(device, REG_SCRATCH_UMSK,
GSL_RB_MEMPTRS_SCRATCH_MASK);
/* update the eoptimestamp field with the last retired timestamp */
kgsl_sharedmem_writel(&device->memstore,
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp),
rb->timestamp);
/* load the CP ucode */
status = adreno_ringbuffer_load_pm4_ucode(device);
if (status != 0)
return status;
/* load the prefetch parser ucode */
status = adreno_ringbuffer_load_pfp_ucode(device);
if (status != 0)
return status;
adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804);
rb->rptr = 0;
rb->wptr = 0;
/* clear ME_HALT to start micro engine */
adreno_regwrite(device, REG_CP_ME_CNTL, 0);
/* ME_INIT */
cmds = adreno_ringbuffer_allocspace(rb, 19);
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-19);
GSL_RB_WRITE(cmds, cmds_gpu, CP_HDR_ME_INIT);
/* All fields present (bits 9:0) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x000003ff);
/* Disable/Enable Real-Time Stream processing (present but ignored) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Enable (2D <-> 3D) implicit synchronization (present but ignored) */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL));
GSL_RB_WRITE(cmds, cmds_gpu,
SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE));
/* Instruction memory size: */
GSL_RB_WRITE(cmds, cmds_gpu,
(adreno_encode_istore_size(adreno_dev)
| adreno_dev->pix_shader_start));
/* Maximum Contexts */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000001);
/* Write Confirm Interval and The CP will wait the
* wait_interval * 16 clocks between polling */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* NQ and External Memory Swap */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Protected mode error checking */
GSL_RB_WRITE(cmds, cmds_gpu, GSL_RB_PROTECTED_MODE_CONTROL);
/* Disable header dumping and Header dump address */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
/* Header dump size */
GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000);
adreno_ringbuffer_submit(rb);
/* idle device to validate ME INIT */
status = adreno_idle(device);
if (status == 0)
rb->flags |= KGSL_FLAGS_STARTED;
return status;
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* start-of-pipeline and end-of-pipeline timestamps */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += 2;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
total_sizedwords += 2;
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
total_sizedwords += 3;
total_sizedwords += 4;
total_sizedwords += 3;
} else {
total_sizedwords += 4;
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
if (context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp[context_id];
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (adreno_is_a3xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
/**
* _ringbuffer_bootstrap_ucode() - Bootstrap GPU Ucode
* @rb: Pointer to adreno ringbuffer
* @load_jt: If non zero only load Jump tables
*
* Bootstrap ucode for GPU
* load_jt == 0, bootstrap full microcode
* load_jt == 1, bootstrap jump tables of microcode
*
* For example a bootstrap packet would like below
* Setup a type3 bootstrap packet
* PFP size to bootstrap
* PFP addr to write the PFP data
* PM4 size to bootstrap
* PM4 addr to write the PM4 data
* PFP dwords from microcode to bootstrap
* PM4 size dwords from microcode to bootstrap
*/
static int _ringbuffer_bootstrap_ucode(struct adreno_ringbuffer *rb,
unsigned int load_jt)
{
unsigned int *cmds, bootstrap_size, rb_size;
int i = 0;
int ret;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int pm4_size, pm4_idx, pm4_addr, pfp_size, pfp_idx, pfp_addr;
/* Only bootstrap jump tables of ucode */
if (load_jt) {
pm4_idx = adreno_dev->gpucore->pm4_jt_idx;
pm4_addr = adreno_dev->gpucore->pm4_jt_addr;
pfp_idx = adreno_dev->gpucore->pfp_jt_idx;
pfp_addr = adreno_dev->gpucore->pfp_jt_addr;
} else {
/* Bootstrap full ucode */
pm4_idx = 1;
pm4_addr = 0;
pfp_idx = 1;
pfp_addr = 0;
}
pm4_size = (adreno_dev->pm4_fw_size - pm4_idx);
pfp_size = (adreno_dev->pfp_fw_size - pfp_idx);
bootstrap_size = (pm4_size + pfp_size + 5);
/*
* Overwrite the first entry in the jump table with the special
* bootstrap opcode
*/
if (adreno_is_a4xx(adreno_dev)) {
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR,
0x400);
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA,
0x6f0009);
/*
* The support packets (the RMW and INTERRUPT) that are sent
* after the bootstrap packet should not be included in the size
* of the bootstrap packet but we do need to reserve enough
* space for those too
*/
rb_size = bootstrap_size + 6;
} else {
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR,
0x200);
adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA,
0x6f0005);
rb_size = bootstrap_size;
}
/* clear ME_HALT to start micro engine */
adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0);
cmds = adreno_ringbuffer_allocspace(rb, rb_size);
if (IS_ERR(cmds))
return PTR_ERR(cmds);
if (cmds == NULL)
return -ENOSPC;
/* Construct the packet that bootsraps the ucode */
*cmds++ = cp_type3_packet(CP_BOOTSTRAP_UCODE, (bootstrap_size - 1));
*cmds++ = pfp_size;
*cmds++ = pfp_addr;
*cmds++ = pm4_size;
*cmds++ = pm4_addr;
/**
* Theory of operation:
*
* In A4x, we cannot have the PFP executing instructions while its instruction
* RAM is loading. We load the PFP's instruction RAM using type-0 writes
* from the ME.
*
* To make sure the PFP is not fetching instructions at the same time,
* we put it in a one-instruction loop:
* mvc (ME), (ringbuffer)
* which executes repeatedly until all of the data has been moved from
* the ring buffer to the ME.
*/
if (adreno_is_a4xx(adreno_dev)) {
for (i = pm4_idx; i < adreno_dev->pm4_fw_size; i++)
*cmds++ = adreno_dev->pm4_fw[i];
for (i = pfp_idx; i < adreno_dev->pfp_fw_size; i++)
*cmds++ = adreno_dev->pfp_fw[i];
*cmds++ = cp_type3_packet(CP_REG_RMW, 3);
*cmds++ = 0x20000000 + A4XX_CP_RB_WPTR;
*cmds++ = 0xffffffff;
*cmds++ = 0x00000002;
*cmds++ = cp_type3_packet(CP_INTERRUPT, 1);
*cmds++ = 0;
rb->wptr = rb->wptr - 2;
adreno_ringbuffer_submit(rb, NULL);
rb->wptr = rb->wptr + 2;
} else {
for (i = pfp_idx; i < adreno_dev->pfp_fw_size; i++)
*cmds++ = adreno_dev->pfp_fw[i];
for (i = pm4_idx; i < adreno_dev->pm4_fw_size; i++)
*cmds++ = adreno_dev->pm4_fw[i];
adreno_ringbuffer_submit(rb, NULL);
}
/* idle device to validate bootstrap */
ret = adreno_spin_idle(device);
if (ret) {
KGSL_DRV_ERR(rb->device,
"microcode bootstrap failed to idle\n");
kgsl_device_snapshot(device, NULL);
}
/* Clear the chicken bit for speed up on A430 and its derivatives */
if (!adreno_is_a420(adreno_dev))
kgsl_regwrite(device, A4XX_CP_DEBUG,
A4XX_CP_DEBUG_DEFAULT & ~(1 << 14));
return ret;
}