static int kgsl_iommu_init(struct kgsl_mmu *mmu)
{
int status = 0;
struct kgsl_iommu *iommu;
iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL);
if (!iommu) {
KGSL_CORE_ERR("kzalloc(%d) failed\n",
sizeof(struct kgsl_iommu));
return -ENOMEM;
}
mmu->priv = iommu;
status = kgsl_get_iommu_ctxt(mmu);
if (status)
goto done;
status = kgsl_set_register_map(mmu);
if (status)
goto done;
kgsl_sharedmem_writel(&mmu->setstate_memory,
KGSL_IOMMU_SETSTATE_NOP_OFFSET,
cp_nop_packet(1));
dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n",
__func__);
done:
if (status) {
kfree(iommu);
mmu->priv = NULL;
}
return status;
}
/**
* adreno_context_restore() - generic context restore handler
* @adreno_dev: the device
* @context: the context
*
* Basic context restore handler that writes the context identifier
* to the ringbuffer and issues pagetable switch commands if necessary.
*/
static int adreno_context_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device;
unsigned int cmds[8];
if (adreno_dev == NULL || context == NULL)
return -EINVAL;
device = &adreno_dev->dev;
/* write the context identifier to the ringbuffer */
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
cmds[4] = context->base.id;
/* Flush the UCHE for new context */
cmds[5] = cp_type0_packet(
adreno_getreg(adreno_dev, ADRENO_REG_UCHE_INVALIDATE0), 2);
cmds[6] = 0;
if (adreno_is_a4xx(adreno_dev))
cmds[7] = 0x12;
else if (adreno_is_a3xx(adreno_dev))
cmds[7] = 0x90000000;
return adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, cmds, 8);
}
static void
adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb, unsigned int numcmds,
int wptr_ahead)
{
int nopcount;
unsigned int freecmds;
unsigned int *cmds;
uint cmds_gpu;
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned long wait_timeout = msecs_to_jiffies(adreno_dev->wait_timeout);
unsigned long wait_time;
/* if wptr ahead, fill the remaining with NOPs */
if (wptr_ahead) {
/* -1 for header */
nopcount = rb->sizedwords - rb->wptr - 1;
cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr;
GSL_RB_WRITE(cmds, cmds_gpu, cp_nop_packet(nopcount));
/* Make sure that rptr is not 0 before submitting
* commands at the end of ringbuffer. We do not
* want the rptr and wptr to become equal when
* the ringbuffer is not empty */
do {
GSL_RB_GET_READPTR(rb, &rb->rptr);
} while (!rb->rptr);
rb->wptr++;
adreno_ringbuffer_submit(rb);
rb->wptr = 0;
}
wait_time = jiffies + wait_timeout;
/* wait for space in ringbuffer */
while (1) {
GSL_RB_GET_READPTR(rb, &rb->rptr);
freecmds = rb->rptr - rb->wptr;
if (freecmds == 0 || freecmds > numcmds)
break;
if (time_after(jiffies, wait_time)) {
KGSL_DRV_ERR(rb->device,
"Timed out while waiting for freespace in ringbuffer "
"rptr: 0x%x, wptr: 0x%x\n", rb->rptr, rb->wptr);
if (!adreno_dump_and_recover(rb->device))
wait_time = jiffies + wait_timeout;
else
/* GPU is hung and we cannot recover */
BUG();
}
}
}
static void a2xx_ctxt_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device = &adreno_dev->dev;
unsigned int cmds[5];
if (context == NULL) {
/* No context - set the default apgetable and thats it */
kgsl_mmu_setstate(device, device->mmu.defaultpagetable);
return;
}
KGSL_CTXT_INFO(device, "context flags %08x\n", context->flags);
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(current_context);
cmds[4] = (unsigned int) context;
adreno_ringbuffer_issuecmds(device, 0, cmds, 5);
kgsl_mmu_setstate(device, context->pagetable);
#ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP
kgsl_cffdump_syncmem(NULL, &context->gpustate,
context->gpustate.gpuaddr, LCC_SHADOW_SIZE +
REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, false);
#endif
/* restore gmem.
* (note: changes shader. shader must not already be restored.)
*/
if (context->flags & CTXT_FLAGS_GMEM_RESTORE) {
adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE,
context->context_gmem_shadow.gmem_restore, 3);
/* Restore TP0_CHICKEN */
adreno_ringbuffer_issuecmds(device, 0,
context->chicken_restore, 3);
context->flags &= ~CTXT_FLAGS_GMEM_RESTORE;
}
/* restore registers and constants. */
adreno_ringbuffer_issuecmds(device, 0,
context->reg_restore, 3);
/* restore shader instructions & partitioning. */
if (context->flags & CTXT_FLAGS_SHADER_RESTORE) {
adreno_ringbuffer_issuecmds(device, 0,
context->shader_restore, 3);
}
if (adreno_is_a20x(adreno_dev)) {
cmds[0] = cp_type3_packet(CP_SET_BIN_BASE_OFFSET, 1);
cmds[1] = context->bin_base_offset;
adreno_ringbuffer_issuecmds(device, 0, cmds, 2);
}
}
static int
adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb,
unsigned int numcmds, int wptr_ahead)
{
int nopcount;
unsigned int freecmds;
unsigned int *cmds;
unsigned int gpuaddr;
unsigned long wait_time;
unsigned long wait_timeout = msecs_to_jiffies(ADRENO_IDLE_TIMEOUT);
unsigned long wait_time_part;
unsigned int rptr;
/* if wptr ahead, fill the remaining with NOPs */
if (wptr_ahead) {
/* -1 for header */
nopcount = KGSL_RB_DWORDS - rb->wptr - 1;
cmds = RB_HOSTPTR(rb, rb->wptr);
gpuaddr = RB_GPUADDR(rb, rb->wptr);
*cmds = cp_nop_packet(nopcount);
kgsl_cffdump_write(rb->device, gpuaddr, *cmds);
/* Make sure that rptr is not 0 before submitting
* commands at the end of ringbuffer. We do not
* want the rptr and wptr to become equal when
* the ringbuffer is not empty */
do {
rptr = adreno_get_rptr(rb);
} while (!rptr);
rb->wptr = 0;
}
wait_time = jiffies + wait_timeout;
wait_time_part = jiffies + msecs_to_jiffies(KGSL_TIMEOUT_PART);
/* wait for space in ringbuffer */
while (1) {
rptr = adreno_get_rptr(rb);
freecmds = rptr - rb->wptr;
if (freecmds == 0 || freecmds > numcmds)
break;
if (time_after(jiffies, wait_time)) {
KGSL_DRV_ERR(rb->device,
"Timed out while waiting for freespace in ringbuffer "
"rptr: 0x%x, wptr: 0x%x\n", rptr, rb->wptr);
return -ETIMEDOUT;
}
}
return 0;
}
static int kgsl_iommu_init(struct kgsl_mmu *mmu)
{
/*
* intialize device mmu
*
* call this with the global lock held
*/
int status = 0;
struct kgsl_iommu *iommu;
iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL);
if (!iommu) {
KGSL_CORE_ERR("kzalloc(%d) failed\n",
sizeof(struct kgsl_iommu));
return -ENOMEM;
}
mmu->priv = iommu;
status = kgsl_get_iommu_ctxt(mmu);
if (status)
goto done;
status = kgsl_set_register_map(mmu);
if (status)
goto done;
iommu->iommu_reg_list = kgsl_iommuv1_reg;
iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1;
if (msm_soc_version_supports_iommu_v1()) {
iommu->iommu_reg_list = kgsl_iommuv1_reg;
iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V1;
} else {
iommu->iommu_reg_list = kgsl_iommuv2_reg;
iommu->ctx_offset = KGSL_IOMMU_CTX_OFFSET_V2;
}
/* A nop is required in an indirect buffer when switching
* pagetables in-stream */
kgsl_sharedmem_writel(&mmu->setstate_memory,
KGSL_IOMMU_SETSTATE_NOP_OFFSET,
cp_nop_packet(1));
dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n",
__func__);
done:
if (status) {
kfree(iommu);
mmu->priv = NULL;
}
return status;
}
static int kgsl_iommu_init(struct kgsl_mmu *mmu)
{
/*
* intialize device mmu
*
* call this with the global lock held
*/
int status = 0;
struct kgsl_iommu *iommu;
iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL);
if (!iommu) {
KGSL_CORE_ERR("kzalloc(%d) failed\n",
sizeof(struct kgsl_iommu));
return -ENOMEM;
}
iommu->asids = kzalloc(BITS_TO_LONGS(KGSL_IOMMU_MAX_ASIDS) *
sizeof(unsigned long), GFP_KERNEL);
if (!iommu->asids) {
KGSL_CORE_ERR("kzalloc(%d) failed\n",
sizeof(struct kgsl_iommu));
status = -ENOMEM;
goto done;
}
mmu->priv = iommu;
status = kgsl_get_iommu_ctxt(mmu);
if (status)
goto done;
status = kgsl_set_register_map(mmu);
if (status)
goto done;
/* A nop is required in an indirect buffer when switching
* pagetables in-stream */
kgsl_sharedmem_writel(&mmu->setstate_memory,
KGSL_IOMMU_SETSTATE_NOP_OFFSET,
cp_nop_packet(1));
dev_info(mmu->device->dev, "|%s| MMU type set for device is IOMMU\n",
__func__);
done:
if (status) {
kfree(iommu->asids);
kfree(iommu);
mmu->priv = NULL;
}
return status;
}
/**
* adreno_context_restore() - generic context restore handler
* @rb: The RB in which context is to be restored
*
* Basic context restore handler that writes the context identifier
* to the ringbuffer and issues pagetable switch commands if necessary.
*/
static void adreno_context_restore(struct adreno_ringbuffer *rb)
{
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_context *drawctxt = rb->drawctxt_active;
unsigned int cmds[11];
int ret;
if (!drawctxt)
return;
/*
* write the context identifier to the ringbuffer, write to both
* the global index and the index of the RB in which the context
* operates. The global values will always be reliable since we
* could be in middle of RB switch in which case the RB value may
* not be accurate
*/
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_RB_OFFSET(rb, current_context);
cmds[4] = drawctxt->base.id;
cmds[5] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[6] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
current_context);
cmds[7] = drawctxt->base.id;
/* Flush the UCHE for new context */
cmds[8] = cp_type0_packet(
adreno_getreg(adreno_dev, ADRENO_REG_UCHE_INVALIDATE0), 2);
cmds[9] = 0;
if (adreno_is_a4xx(adreno_dev))
cmds[10] = 0x12;
else if (adreno_is_a3xx(adreno_dev))
cmds[10] = 0x90000000;
ret = adreno_ringbuffer_issuecmds(rb, KGSL_CMD_FLAGS_NONE, cmds, 11);
if (ret) {
/*
* A failure to submit commands to ringbuffer means RB may
* be full, in this case wait for idle and use CPU
*/
ret = adreno_idle(device);
BUG_ON(ret);
_adreno_context_restore_cpu(rb, drawctxt);
}
}
static void
adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb, unsigned int numcmds,
int wptr_ahead)
{
int nopcount;
unsigned int freecmds;
unsigned int *cmds;
uint cmds_gpu;
/* if wptr ahead, fill the remaining with NOPs */
if (wptr_ahead) {
/* -1 for header */
nopcount = rb->sizedwords - rb->wptr - 1;
cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr;
GSL_RB_WRITE(cmds, cmds_gpu, cp_nop_packet(nopcount));
/* Make sure that rptr is not 0 before submitting
* commands at the end of ringbuffer. We do not
* want the rptr and wptr to become equal when
* the ringbuffer is not empty */
do {
GSL_RB_GET_READPTR(rb, &rb->rptr);
} while (!rb->rptr);
rb->wptr++;
adreno_ringbuffer_submit(rb);
rb->wptr = 0;
}
/* wait for space in ringbuffer */
do {
GSL_RB_GET_READPTR(rb, &rb->rptr);
freecmds = rb->rptr - rb->wptr;
} while ((freecmds != 0) && (freecmds <= numcmds));
}
int adreno_context_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device;
unsigned int cmds[5];
if (adreno_dev == NULL || context == NULL)
return -EINVAL;
device = &adreno_dev->dev;
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
cmds[4] = context->base.id;
return adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, cmds, 5);
}
static void a2xx_drawctxt_restore(struct adreno_device *adreno_dev,
struct adreno_context *context)
{
struct kgsl_device *device = &adreno_dev->dev;
unsigned int cmds[5];
if (context == NULL) {
kgsl_mmu_setstate(&device->mmu, device->mmu.defaultpagetable,
adreno_dev->drawctxt_active->id);
return;
}
KGSL_CTXT_INFO(device, "context flags %08x\n", context->flags);
cmds[0] = cp_nop_packet(1);
cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER;
cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2);
cmds[3] = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, current_context);
cmds[4] = context->id;
adreno_ringbuffer_issuecmds(device, context, KGSL_CMD_FLAGS_NONE,
cmds, 5);
kgsl_mmu_setstate(&device->mmu, context->pagetable, context->id);
#ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP
kgsl_cffdump_syncmem(NULL, &context->gpustate,
context->gpustate.gpuaddr, LCC_SHADOW_SIZE +
REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, false);
#endif
if (context->flags & CTXT_FLAGS_GMEM_RESTORE) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_PMODE,
context->context_gmem_shadow.gmem_restore, 3);
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE,
context->chicken_restore, 3);
}
context->flags &= ~CTXT_FLAGS_GMEM_RESTORE;
}
if (!(context->flags & CTXT_FLAGS_PREAMBLE)) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, context->reg_restore, 3);
if (context->flags & CTXT_FLAGS_SHADER_RESTORE) {
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE,
context->shader_restore, 3);
}
}
if (adreno_is_a20x(adreno_dev)) {
cmds[0] = cp_type3_packet(CP_SET_BIN_BASE_OFFSET, 1);
cmds[1] = context->bin_base_offset;
adreno_ringbuffer_issuecmds(device, context,
KGSL_CMD_FLAGS_NONE, cmds, 2);
}
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
if (context != NULL) {
/*
* if the context was not created with per context timestamp
* support, we must use the global timestamp since issueibcmds
* will be returning that one.
*/
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
}
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
total_sizedwords += 2; /* scratchpad ts for recovery */
if (context) {
total_sizedwords += 3; /* sop timestamp */
total_sizedwords += 4; /* eop timestamp */
total_sizedwords += 3; /* global timestamp without cache
* flush for non-zero context */
} else {
total_sizedwords += 4; /* global timestamp for recovery*/
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
/* always increment the global timestamp. once. */
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
/* scratchpad ts for recovery */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (adreno_is_a3xx(adreno_dev)) {
/*
* FLush HLSQ lazy updates to make sure there are no
* rsources pending for indirect loads after the timestamp
*/
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
if (context) {
/* start-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* end-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static void
adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb, unsigned int numcmds,
int wptr_ahead)
{
int nopcount;
unsigned int freecmds;
unsigned int *cmds;
uint cmds_gpu;
unsigned long wait_time;
unsigned long wait_timeout = msecs_to_jiffies(ADRENO_IDLE_TIMEOUT);
unsigned long wait_time_part;
unsigned int prev_reg_val[hang_detect_regs_count];
memset(prev_reg_val, 0, sizeof(prev_reg_val));
/* if wptr ahead, fill the remaining with NOPs */
if (wptr_ahead) {
/* -1 for header */
nopcount = rb->sizedwords - rb->wptr - 1;
cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr;
GSL_RB_WRITE(cmds, cmds_gpu, cp_nop_packet(nopcount));
/* Make sure that rptr is not 0 before submitting
* commands at the end of ringbuffer. We do not
* want the rptr and wptr to become equal when
* the ringbuffer is not empty */
do {
GSL_RB_GET_READPTR(rb, &rb->rptr);
} while (!rb->rptr);
rb->wptr++;
adreno_ringbuffer_submit(rb);
rb->wptr = 0;
}
wait_time = jiffies + wait_timeout;
wait_time_part = jiffies + msecs_to_jiffies(KGSL_TIMEOUT_PART);
/* wait for space in ringbuffer */
while (1) {
GSL_RB_GET_READPTR(rb, &rb->rptr);
freecmds = rb->rptr - rb->wptr;
if (freecmds == 0 || freecmds > numcmds)
break;
/* Dont wait for timeout, detect hang faster.
*/
if (time_after(jiffies, wait_time_part)) {
wait_time_part = jiffies +
msecs_to_jiffies(KGSL_TIMEOUT_PART);
if ((adreno_hang_detect(rb->device,
prev_reg_val))){
KGSL_DRV_ERR(rb->device,
"Hang detected while waiting for freespace in"
"ringbuffer rptr: 0x%x, wptr: 0x%x\n",
rb->rptr, rb->wptr);
goto err;
}
}
if (time_after(jiffies, wait_time)) {
KGSL_DRV_ERR(rb->device,
"Timed out while waiting for freespace in ringbuffer "
"rptr: 0x%x, wptr: 0x%x\n", rb->rptr, rb->wptr);
goto err;
}
continue;
err:
if (!adreno_dump_and_recover(rb->device))
wait_time = jiffies + wait_timeout;
else
/* GPU is hung and we cannot recover */
BUG();
}
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += 2;
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 7;
total_sizedwords += 2;
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
total_sizedwords += 3;
total_sizedwords += 4;
total_sizedwords += 3;
} else {
total_sizedwords += 4;
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
if (context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp[context_id];
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (adreno_is_a3xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
/* Snapshot the ringbuffer memory */
static int snapshot_rb(struct kgsl_device *device, void *snapshot,
int remain, void *priv)
{
struct kgsl_snapshot_rb *header = snapshot;
unsigned int *data = snapshot + sizeof(*header);
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
unsigned int rbbase, ptbase, rptr, *rbptr;
int start, stop, index;
int numitems, size;
int parse_ibs = 0, ib_parse_start;
/* Get the GPU address of the ringbuffer */
kgsl_regread(device, REG_CP_RB_BASE, &rbbase);
/* Get the physical address of the MMU pagetable */
ptbase = kgsl_mmu_get_current_ptbase(device);
/* Get the current read pointers for the RB */
kgsl_regread(device, REG_CP_RB_RPTR, &rptr);
/* start the dump at the rptr minus some history */
start = (int) rptr - NUM_DWORDS_OF_RINGBUFFER_HISTORY;
if (start < 0)
start += rb->sizedwords;
/*
* Stop the dump at the point where the software last wrote. Don't use
* the hardware value here on the chance that it didn't get properly
* updated
*/
stop = (int) rb->wptr + 16;
if (stop > rb->sizedwords)
stop -= rb->sizedwords;
/* Set up the header for the section */
numitems = (stop > start) ? stop - start :
(rb->sizedwords - start) + stop;
size = (numitems << 2);
if (remain < size + sizeof(*header)) {
KGSL_DRV_ERR(device,
"snapshot: Not enough memory for the rb section");
return 0;
}
/* Write the sub-header for the section */
header->start = start;
header->end = stop;
header->wptr = rb->wptr;
header->rbsize = rb->sizedwords;
header->count = numitems;
/*
* We can only reliably dump IBs from the beginning of the context,
* and it turns out that for the vast majority of the time we really
* only care about the current context when it comes to diagnosing
* a hang. So, with an eye to limiting the buffer dumping to what is
* really useful find the beginning of the context and only dump
* IBs from that point
*/
index = rptr;
ib_parse_start = start;
rbptr = rb->buffer_desc.hostptr;
while (index != start) {
index--;
if (index < 0) {
/*
* The marker we are looking for is 2 dwords long, so
* when wrapping, go back 2 from the end so we don't
* access out of range in the if statement below
*/
index = rb->sizedwords - 2;
/*
* Account for the possibility that start might be at
* rb->sizedwords - 1
*/
if (start == rb->sizedwords - 1)
break;
}
/*
* Look for a NOP packet with the context switch identifier in
* the second dword
*/
if (rbptr[index] == cp_nop_packet(1) &&
rbptr[index + 1] == KGSL_CONTEXT_TO_MEM_IDENTIFIER) {
ib_parse_start = index;
break;
}
}
index = start;
/*
* Loop through the RB, copying the data and looking for indirect
* buffers and MMU pagetable changes
*/
while (index != rb->wptr) {
*data = rbptr[index];
/* Only parse IBs between the context start and the rptr */
if (index == ib_parse_start)
parse_ibs = 1;
if (index == rptr)
parse_ibs = 0;
if (parse_ibs && adreno_cmd_is_ib(rbptr[index]))
push_object(device, SNAPSHOT_OBJ_TYPE_IB, ptbase,
rbptr[index + 1], rbptr[index + 2]);
index = index + 1;
if (index == rb->sizedwords)
index = 0;
data++;
}
/* Dump 16 dwords past the wptr, but don't bother interpeting it */
while (index != stop) {
*data = rbptr[index];
index = index + 1;
if (index == rb->sizedwords)
index = 0;
data++;
}
/* Return the size of the section */
return size + sizeof(*header);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int rcmd_gpu;
unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
unsigned int gpuaddr = rb->device->memstore.gpuaddr;
/*
* if the context was not created with per context timestamp
* support, we must use the global timestamp since issueibcmds
* will be returning that one.
*/
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
context_id = context->id;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
if (adreno_is_a2xx(adreno_dev))
total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */
total_sizedwords += 2; /* scratchpad ts for recovery */
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
total_sizedwords += 3; /* sop timestamp */
total_sizedwords += 4; /* eop timestamp */
total_sizedwords += 3; /* global timestamp without cache
* flush for non-zero context */
} else {
total_sizedwords += 4; /* global timestamp for recovery*/
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
/* GPU may hang during space allocation, if thats the case the current
* context may have hung the GPU */
if (context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp[context_id];
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
/* always increment the global timestamp. once. */
rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
if (context) {
if (context_id == KGSL_MEMSTORE_GLOBAL)
rb->timestamp[context_id] =
rb->timestamp[KGSL_MEMSTORE_GLOBAL];
else
rb->timestamp[context_id]++;
}
timestamp = rb->timestamp[context_id];
/* HW Workaround for MMU Page fault
* due to memory getting free early before
* GPU completes it.
*/
if (adreno_is_a2xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
/* scratchpad ts for recovery */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS) {
/* start-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* end-of-pipeline timestamp */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_MEM_WRITE, 2));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
} else {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu,
rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
KGSL_MEMSTORE_OFFSET(
context_id, ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* start-of-pipeline and end-of-pipeline timestamps */
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static uint32_t
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
struct adreno_context *context,
unsigned int flags, unsigned int *cmds,
int sizedwords)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned int *ringcmds;
unsigned int timestamp;
unsigned int total_sizedwords = sizedwords + 6;
unsigned int i;
unsigned int rcmd_gpu;
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
if (adreno_is_a2xx(adreno_dev))
total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */
if (adreno_is_a20x(adreno_dev))
total_sizedwords += 2; /* CACHE_FLUSH */
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
/* GPU may hang during space allocation, if thats the case the current
* context may have hung the GPU */
if (context && context->flags & CTXT_FLAGS_GPU_HANG) {
KGSL_CTXT_WARN(rb->device,
"Context %p caused a gpu hang. Will not accept commands for context %d\n",
context, context->id);
return rb->timestamp;
}
rcmd_gpu = rb->buffer_desc.gpuaddr
+ sizeof(uint)*(rb->wptr-total_sizedwords);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
}
for (i = 0; i < sizedwords; i++) {
GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
cmds++;
}
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
}
rb->timestamp++;
timestamp = rb->timestamp;
/* HW Workaround for MMU Page fault
* due to memory getting free early before
* GPU completes it.
*/
if (adreno_is_a2xx(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
}
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
(rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)));
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
if (adreno_is_a20x(adreno_dev)) {
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_EVENT_WRITE, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH);
}
if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
/* Conditional execution based on memory values */
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_COND_EXEC, 4));
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr +
KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp);
/* # of conditional command DWORDs */
GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
GSL_RB_WRITE(ringcmds, rcmd_gpu,
cp_type3_packet(CP_INTERRUPT, 1));
GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
}
static void
adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb, unsigned int numcmds,
int wptr_ahead)
{
int nopcount;
unsigned int freecmds;
unsigned int *cmds;
uint cmds_gpu;
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
unsigned long wait_timeout = msecs_to_jiffies(adreno_dev->wait_timeout);
unsigned long wait_time;
unsigned long wait_time_part;
unsigned int msecs_part = KGSL_TIMEOUT_PART;
unsigned int prev_reg_val[hang_detect_regs_count];
memset(prev_reg_val, 0, sizeof(prev_reg_val));
if (wptr_ahead) {
nopcount = rb->sizedwords - rb->wptr - 1;
cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr;
GSL_RB_WRITE(cmds, cmds_gpu, cp_nop_packet(nopcount));
do {
GSL_RB_GET_READPTR(rb, &rb->rptr);
} while (!rb->rptr);
rb->wptr++;
adreno_ringbuffer_submit(rb);
rb->wptr = 0;
}
wait_time = jiffies + wait_timeout;
wait_time_part = jiffies + msecs_to_jiffies(msecs_part);
while (1) {
GSL_RB_GET_READPTR(rb, &rb->rptr);
freecmds = rb->rptr - rb->wptr;
if (freecmds == 0 || freecmds > numcmds)
break;
if (time_after(jiffies, wait_time_part)) {
wait_time_part = jiffies +
msecs_to_jiffies(msecs_part);
if ((adreno_hang_detect(rb->device,
prev_reg_val))){
KGSL_DRV_ERR(rb->device,
"Hang detected while waiting for freespace in"
"ringbuffer rptr: 0x%x, wptr: 0x%x\n",
rb->rptr, rb->wptr);
goto err;
}
}
if (time_after(jiffies, wait_time)) {
KGSL_DRV_ERR(rb->device,
"Timed out while waiting for freespace in ringbuffer "
"rptr: 0x%x, wptr: 0x%x\n", rb->rptr, rb->wptr);
goto err;
}
continue;
err:
if (!adreno_dump_and_recover(rb->device)) {
wait_time = jiffies + wait_timeout;
} else {
BUG();
}
}
}
