static uint32_t kgsl_ringbuffer_addcmds(struct kgsl_ringbuffer *rb, unsigned int flags, unsigned int *cmds, int sizedwords) { unsigned int *ringcmds; unsigned int timestamp; unsigned int total_sizedwords = sizedwords + 6; total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 9 : 0; ringcmds = kgsl_ringbuffer_allocspace(rb, total_sizedwords); if (flags & KGSL_CMD_FLAGS_PMODE) { *ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1); *ringcmds++ = 0; } memcpy(ringcmds, cmds, (sizedwords << 2)); ringcmds += sizedwords; if (flags & KGSL_CMD_FLAGS_PMODE) { *ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1); *ringcmds++ = 1; } rb->timestamp++; timestamp = rb->timestamp; *ringcmds++ = pm4_type0_packet(REG_CP_TIMESTAMP, 1); *ringcmds++ = rb->timestamp; *ringcmds++ = pm4_type3_packet(PM4_EVENT_WRITE, 3); *ringcmds++ = CACHE_FLUSH_TS; *ringcmds++ = (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)); *ringcmds++ = rb->timestamp; if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) { *ringcmds++ = pm4_type3_packet(PM4_COND_EXEC, 4); *ringcmds++ = (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2; *ringcmds++ = (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2; *ringcmds++ = rb->timestamp; *ringcmds++ = 4; *ringcmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); *ringcmds++ = 0x00000000; *ringcmds++ = pm4_type3_packet(PM4_INTERRUPT, 1); *ringcmds++ = CP_INT_CNTL__RB_INT_MASK; }
static void a2xx_ctxt_restore(struct adreno_device *adreno_dev, struct adreno_context *context) { struct kgsl_device *device = &adreno_dev->dev; unsigned int cmds[5]; if (context == NULL) { /* No context - set the default apgetable and thats it */ kgsl_mmu_setstate(device, device->mmu.defaultpagetable); return; } KGSL_CTXT_INFO(device, "context flags %08x\n", context->flags); cmds[0] = cp_nop_packet(1); cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER; cmds[2] = cp_type3_packet(CP_MEM_WRITE, 2); cmds[3] = device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(current_context); cmds[4] = (unsigned int) context; adreno_ringbuffer_issuecmds(device, 0, cmds, 5); kgsl_mmu_setstate(device, context->pagetable); #ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP kgsl_cffdump_syncmem(NULL, &context->gpustate, context->gpustate.gpuaddr, LCC_SHADOW_SIZE + REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, false); #endif /* restore gmem. * (note: changes shader. shader must not already be restored.) */ if (context->flags & CTXT_FLAGS_GMEM_RESTORE) { adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE, context->context_gmem_shadow.gmem_restore, 3); /* Restore TP0_CHICKEN */ adreno_ringbuffer_issuecmds(device, 0, context->chicken_restore, 3); context->flags &= ~CTXT_FLAGS_GMEM_RESTORE; } /* restore registers and constants. */ adreno_ringbuffer_issuecmds(device, 0, context->reg_restore, 3); /* restore shader instructions & partitioning. */ if (context->flags & CTXT_FLAGS_SHADER_RESTORE) { adreno_ringbuffer_issuecmds(device, 0, context->shader_restore, 3); } if (adreno_is_a20x(adreno_dev)) { cmds[0] = cp_type3_packet(CP_SET_BIN_BASE_OFFSET, 1); cmds[1] = context->bin_base_offset; adreno_ringbuffer_issuecmds(device, 0, cmds, 2); } }
static uint32_t adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, struct adreno_context *context, unsigned int flags, unsigned int *cmds, int sizedwords) { struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device); unsigned int *ringcmds; unsigned int timestamp; unsigned int total_sizedwords = sizedwords + 6; unsigned int i; unsigned int rcmd_gpu; /* reserve space to temporarily turn off protected mode * error checking if needed */ total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0; /* 2 dwords to store the start of command sequence */ total_sizedwords += 2; if (adreno_is_a2xx(adreno_dev)) total_sizedwords += 2; /* CP_WAIT_FOR_IDLE */ if (adreno_is_a20x(adreno_dev)) total_sizedwords += 2; /* CACHE_FLUSH */ ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords); /* GPU may hang during space allocation, if thats the case the current * context may have hung the GPU */ if (context && context->flags & CTXT_FLAGS_GPU_HANG) { KGSL_CTXT_WARN(rb->device, "Context %p caused a gpu hang. Will not accept commands for context %d\n", context, context->id); return rb->timestamp; } rcmd_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-total_sizedwords); GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER); if (flags & KGSL_CMD_FLAGS_PMODE) { /* disable protected mode error checking */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_SET_PROTECTED_MODE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 0); } for (i = 0; i < sizedwords; i++) { GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds); cmds++; } if (flags & KGSL_CMD_FLAGS_PMODE) { /* re-enable protected mode error checking */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_SET_PROTECTED_MODE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 1); } rb->timestamp++; timestamp = rb->timestamp; /* HW Workaround for MMU Page fault * due to memory getting free early before * GPU completes it. */ if (adreno_is_a2xx(adreno_dev)) { GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_WAIT_FOR_IDLE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00); } GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp))); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); if (adreno_is_a20x(adreno_dev)) { GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH); } if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) { /* Conditional execution based on memory values */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_COND_EXEC, 4)); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); /* # of conditional command DWORDs */ GSL_RB_WRITE(ringcmds, rcmd_gpu, 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_INTERRUPT, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK); }
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram) { int status; /*cp_rb_cntl_u cp_rb_cntl; */ union reg_cp_rb_cntl cp_rb_cntl; unsigned int *cmds, rb_cntl; struct kgsl_device *device = rb->device; struct adreno_device *adreno_dev = ADRENO_DEVICE(device); uint cmds_gpu; if (rb->flags & KGSL_FLAGS_STARTED) return 0; if (init_ram) { rb->timestamp = 0; GSL_RB_INIT_TIMESTAMP(rb); } kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0, sizeof(struct kgsl_rbmemptrs)); kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA, (rb->sizedwords << 2)); adreno_regwrite(device, REG_CP_RB_WPTR_BASE, (rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET)); /* setup WPTR delay */ adreno_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */); /*setup REG_CP_RB_CNTL */ adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl); cp_rb_cntl.val = rb_cntl; /* * The size of the ringbuffer in the hardware is the log2 * representation of the size in quadwords (sizedwords / 2) */ cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1); /* * Specify the quadwords to read before updating mem RPTR. * Like above, pass the log2 representation of the blocksize * in quadwords. */ cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3); cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; /* WPTR polling */ /* mem RPTR writebacks */ cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE; adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val); adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); adreno_regwrite(device, REG_CP_RB_RPTR_ADDR, rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_RPTR_OFFSET); /* explicitly clear all cp interrupts */ adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF); /* setup scratch/timestamp */ adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp)); adreno_regwrite(device, REG_SCRATCH_UMSK, GSL_RB_MEMPTRS_SCRATCH_MASK); /* update the eoptimestamp field with the last retired timestamp */ kgsl_sharedmem_writel(&device->memstore, KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp), rb->timestamp); /* load the CP ucode */ status = adreno_ringbuffer_load_pm4_ucode(device); if (status != 0) return status; /* load the prefetch parser ucode */ status = adreno_ringbuffer_load_pfp_ucode(device); if (status != 0) return status; adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804); rb->rptr = 0; rb->wptr = 0; /* clear ME_HALT to start micro engine */ adreno_regwrite(device, REG_CP_ME_CNTL, 0); /* ME_INIT */ cmds = adreno_ringbuffer_allocspace(rb, 19); cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-19); GSL_RB_WRITE(cmds, cmds_gpu, CP_HDR_ME_INIT); /* All fields present (bits 9:0) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x000003ff); /* Disable/Enable Real-Time Stream processing (present but ignored) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* Enable (2D <-> 3D) implicit synchronization (present but ignored) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL)); GSL_RB_WRITE(cmds, cmds_gpu, SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE)); /* Instruction memory size: */ GSL_RB_WRITE(cmds, cmds_gpu, (adreno_encode_istore_size(adreno_dev) | adreno_dev->pix_shader_start)); /* Maximum Contexts */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000001); /* Write Confirm Interval and The CP will wait the * wait_interval * 16 clocks between polling */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* NQ and External Memory Swap */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* Protected mode error checking */ GSL_RB_WRITE(cmds, cmds_gpu, GSL_RB_PROTECTED_MODE_CONTROL); /* Disable header dumping and Header dump address */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* Header dump size */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); adreno_ringbuffer_submit(rb); /* idle device to validate ME INIT */ status = adreno_idle(device); if (status == 0) rb->flags |= KGSL_FLAGS_STARTED; return status; }
/* functions */ void kgsl_cp_intrcallback(struct kgsl_device *device) { unsigned int status = 0, num_reads = 0, master_status = 0; struct kgsl_yamato_device *yamato_device = KGSL_YAMATO_DEVICE(device); struct kgsl_ringbuffer *rb = &yamato_device->ringbuffer; kgsl_yamato_regread_isr(device, REG_MASTER_INT_SIGNAL, &master_status); while (!status && (num_reads < VALID_STATUS_COUNT_MAX) && (master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) { kgsl_yamato_regread_isr(device, REG_CP_INT_STATUS, &status); kgsl_yamato_regread_isr(device, REG_MASTER_INT_SIGNAL, &master_status); num_reads++; } if (num_reads > 1) KGSL_DRV_WARN(device, "Looped %d times to read REG_CP_INT_STATUS\n", num_reads); if (!status) { if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) { /* This indicates that we could not read CP_INT_STAT. * As a precaution just wake up processes so * they can check their timestamps. Since, we * did not ack any interrupts this interrupt will * be generated again */ KGSL_DRV_WARN(device, "Unable to read CP_INT_STATUS\n"); wake_up_interruptible_all(&device->wait_queue); } else KGSL_DRV_WARN(device, "Spurious interrput detected\n"); return; } if (status & CP_INT_CNTL__RB_INT_MASK) { /* signal intr completion event */ unsigned int enableflag = 0; kgsl_sharedmem_writel(&rb->device->memstore, KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), enableflag); wmb(); KGSL_CMD_WARN(rb->device, "ringbuffer rb interrupt\n"); } if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) { KGSL_CMD_CRIT(rb->device, "ringbuffer TO packet in IB interrupt\n"); kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); } if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) { KGSL_CMD_CRIT(rb->device, "ringbuffer opcode error interrupt\n"); kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); } if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) { KGSL_CMD_CRIT(rb->device, "ringbuffer protected mode error interrupt\n"); kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); } if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) { KGSL_CMD_CRIT(rb->device, "ringbuffer reserved bit error interrupt\n"); kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); } if (status & CP_INT_CNTL__IB_ERROR_MASK) { KGSL_CMD_CRIT(rb->device, "ringbuffer IB error interrupt\n"); kgsl_yamato_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); } if (status & CP_INT_CNTL__SW_INT_MASK) KGSL_CMD_INFO(rb->device, "ringbuffer software interrupt\n"); if (status & CP_INT_CNTL__IB2_INT_MASK) KGSL_CMD_INFO(rb->device, "ringbuffer ib2 interrupt\n"); if (status & (~GSL_CP_INT_MASK)) KGSL_CMD_WARN(rb->device, "bad bits in REG_CP_INT_STATUS %08x\n", status); /* only ack bits we understand */ status &= GSL_CP_INT_MASK; kgsl_yamato_regwrite_isr(device, REG_CP_INT_ACK, status); if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) { KGSL_CMD_WARN(rb->device, "ringbuffer ib1/rb interrupt\n"); wake_up_interruptible_all(&device->wait_queue); atomic_notifier_call_chain(&(device->ts_notifier_list), KGSL_DEVICE_YAMATO, NULL); } }
static uint32_t kgsl_ringbuffer_addcmds(struct kgsl_ringbuffer *rb, unsigned int flags, unsigned int *cmds, int sizedwords) { unsigned int *ringcmds; unsigned int timestamp; unsigned int total_sizedwords = sizedwords + 6; unsigned int i; unsigned int rcmd_gpu; /* reserve space to temporarily turn off protected mode * error checking if needed */ total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0; ringcmds = kgsl_ringbuffer_allocspace(rb, total_sizedwords); rcmd_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-total_sizedwords); if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) { GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_nop_packet(1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER); } if (flags & KGSL_CMD_FLAGS_PMODE) { /* disable protected mode error checking */ GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 0); } for (i = 0; i < sizedwords; i++) { GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds); cmds++; } if (flags & KGSL_CMD_FLAGS_PMODE) { /* re-enable protected mode error checking */ GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 1); } rb->timestamp++; timestamp = rb->timestamp; /* start-of-pipeline and end-of-pipeline timestamps */ GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type0_packet(REG_CP_TIMESTAMP, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_EVENT_WRITE, 3)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp))); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) { /* Conditional execution based on memory values */ GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_COND_EXEC, 4)); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); /* # of conditional command DWORDs */ GSL_RB_WRITE(ringcmds, rcmd_gpu, 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_INTERRUPT, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK); }
static void a2xx_cp_intrcallback(struct kgsl_device *device) { unsigned int status = 0, num_reads = 0, master_status = 0; struct adreno_device *adreno_dev = ADRENO_DEVICE(device); struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; int i; adreno_regread(device, REG_MASTER_INT_SIGNAL, &master_status); while (!status && (num_reads < VALID_STATUS_COUNT_MAX) && (master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) { adreno_regread(device, REG_CP_INT_STATUS, &status); adreno_regread(device, REG_MASTER_INT_SIGNAL, &master_status); num_reads++; } if (num_reads > 1) KGSL_DRV_WARN(device, "Looped %d times to read REG_CP_INT_STATUS\n", num_reads); if (!status) { if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) { /* This indicates that we could not read CP_INT_STAT. * As a precaution just wake up processes so * they can check their timestamps. Since, we * did not ack any interrupts this interrupt will * be generated again */ KGSL_DRV_WARN(device, "Unable to read CP_INT_STATUS\n"); wake_up_interruptible_all(&device->wait_queue); } else KGSL_DRV_WARN(device, "Spurious interrput detected\n"); return; } if (status & CP_INT_CNTL__RB_INT_MASK) { /* signal intr completion event */ unsigned int enableflag = 0; kgsl_sharedmem_writel(&rb->device->memstore, KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), enableflag); wmb(); KGSL_CMD_WARN(rb->device, "ringbuffer rb interrupt\n"); } for (i = 0; i < ARRAY_SIZE(kgsl_cp_error_irqs); i++) { if (status & kgsl_cp_error_irqs[i].mask) { KGSL_CMD_CRIT(rb->device, "%s\n", kgsl_cp_error_irqs[i].message); /* * on fatal errors, turn off the interrupts to * avoid storming. This has the side effect of * forcing a PM dump when the timestamp times out */ kgsl_pwrctrl_irq(rb->device, KGSL_PWRFLAGS_OFF); } } /* only ack bits we understand */ status &= CP_INT_MASK; adreno_regwrite(device, REG_CP_INT_ACK, status); if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) { KGSL_CMD_WARN(rb->device, "ringbuffer ib1/rb interrupt\n"); queue_work(device->work_queue, &device->ts_expired_ws); wake_up_interruptible_all(&device->wait_queue); atomic_notifier_call_chain(&(device->ts_notifier_list), device->id, NULL); } }
/* switch drawing contexts */ void adreno_drawctxt_switch(struct adreno_device *adreno_dev, struct adreno_context *drawctxt, unsigned int flags) { struct adreno_context *active_ctxt = adreno_dev->drawctxt_active; struct kgsl_device *device = &adreno_dev->dev; unsigned int cmds[5]; if (drawctxt) { if (flags & KGSL_CONTEXT_SAVE_GMEM) /* Set the flag in context so that the save is done * when this context is switched out. */ drawctxt->flags |= CTXT_FLAGS_GMEM_SAVE; else /* Remove GMEM saving flag from the context */ drawctxt->flags &= ~CTXT_FLAGS_GMEM_SAVE; } /* already current? */ if (active_ctxt == drawctxt) return; KGSL_CTXT_INFO(device, "from %p to %p flags %d\n", adreno_dev->drawctxt_active, drawctxt, flags); /* save old context*/ if (active_ctxt && active_ctxt->flags & CTXT_FLAGS_GPU_HANG) KGSL_CTXT_WARN(device, "Current active context has caused gpu hang\n"); if (active_ctxt != NULL) { KGSL_CTXT_INFO(device, "active_ctxt flags %08x\n", active_ctxt->flags); /* save registers and constants. */ adreno_ringbuffer_issuecmds(device, 0, active_ctxt->reg_save, 3); if (active_ctxt->flags & CTXT_FLAGS_SHADER_SAVE) { /* save shader partitioning and instructions. */ adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE, active_ctxt->shader_save, 3); /* fixup shader partitioning parameter for * SET_SHADER_BASES. */ adreno_ringbuffer_issuecmds(device, 0, active_ctxt->shader_fixup, 3); active_ctxt->flags |= CTXT_FLAGS_SHADER_RESTORE; } if (active_ctxt->flags & CTXT_FLAGS_GMEM_SAVE && active_ctxt->flags & CTXT_FLAGS_GMEM_SHADOW) { /* save gmem. * (note: changes shader. shader must already be saved.) */ adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE, active_ctxt->context_gmem_shadow.gmem_save, 3); /* Restore TP0_CHICKEN */ adreno_ringbuffer_issuecmds(device, 0, active_ctxt->chicken_restore, 3); active_ctxt->flags |= CTXT_FLAGS_GMEM_RESTORE; } } adreno_dev->drawctxt_active = drawctxt; /* restore new context */ if (drawctxt != NULL) { KGSL_CTXT_INFO(device, "drawctxt flags %08x\n", drawctxt->flags); cmds[0] = pm4_nop_packet(1); cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER; cmds[2] = pm4_type3_packet(PM4_MEM_WRITE, 2); cmds[3] = device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(current_context); cmds[4] = (unsigned int)adreno_dev->drawctxt_active; adreno_ringbuffer_issuecmds(device, 0, cmds, 5); kgsl_mmu_setstate(device, drawctxt->pagetable); #ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP kgsl_cffdump_syncmem(NULL, &drawctxt->gpustate, drawctxt->gpustate.gpuaddr, LCC_SHADOW_SIZE + REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, false); #endif /* restore gmem. * (note: changes shader. shader must not already be restored.) */ if (drawctxt->flags & CTXT_FLAGS_GMEM_RESTORE) { adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE, drawctxt->context_gmem_shadow.gmem_restore, 3); /* Restore TP0_CHICKEN */ adreno_ringbuffer_issuecmds(device, 0, drawctxt->chicken_restore, 3); drawctxt->flags &= ~CTXT_FLAGS_GMEM_RESTORE; } /* restore registers and constants. */ adreno_ringbuffer_issuecmds(device, 0, drawctxt->reg_restore, 3); /* restore shader instructions & partitioning. */ if (drawctxt->flags & CTXT_FLAGS_SHADER_RESTORE) { adreno_ringbuffer_issuecmds(device, 0, drawctxt->shader_restore, 3); } cmds[0] = pm4_type3_packet(PM4_SET_BIN_BASE_OFFSET, 1); cmds[1] = drawctxt->bin_base_offset; if (!adreno_is_a220(adreno_dev)) adreno_ringbuffer_issuecmds(device, 0, cmds, 2); } else kgsl_mmu_setstate(device, device->mmu.defaultpagetable); }
int kgsl_ringbuffer_start(struct kgsl_ringbuffer *rb, unsigned int init_ram) { int status; /*cp_rb_cntl_u cp_rb_cntl; */ union reg_cp_rb_cntl cp_rb_cntl; unsigned int *cmds, rb_cntl; struct kgsl_device *device = rb->device; uint cmds_gpu; KGSL_CMD_VDBG("enter (rb=%p)\n", rb); if (rb->flags & KGSL_FLAGS_STARTED) { KGSL_CMD_VDBG("already started return %d\n", 0); return 0; } if (init_ram) { rb->timestamp = 0; GSL_RB_INIT_TIMESTAMP(rb); } kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0, sizeof(struct kgsl_rbmemptrs)); kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA, (rb->sizedwords << 2)); kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_BASE, (rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET)); /* setup WPTR delay */ kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */); /*setup REG_CP_RB_CNTL */ kgsl_yamato_regread(device, REG_CP_RB_CNTL, &rb_cntl); cp_rb_cntl.val = rb_cntl; /* size of ringbuffer */ cp_rb_cntl.f.rb_bufsz = kgsl_ringbuffer_sizelog2quadwords(rb->sizedwords); /* quadwords to read before updating mem RPTR */ cp_rb_cntl.f.rb_blksz = rb->blksizequadwords; cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; /* WPTR polling */ /* mem RPTR writebacks */ cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE; kgsl_yamato_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val); kgsl_yamato_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); kgsl_yamato_regwrite(device, REG_CP_RB_RPTR_ADDR, rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_RPTR_OFFSET); /* explicitly clear all cp interrupts */ kgsl_yamato_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF); /* setup scratch/timestamp */ kgsl_yamato_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp)); kgsl_yamato_regwrite(device, REG_SCRATCH_UMSK, GSL_RB_MEMPTRS_SCRATCH_MASK); /* load the CP ucode */ status = kgsl_ringbuffer_load_pm4_ucode(device); if (status != 0) { KGSL_DRV_ERR("kgsl_ringbuffer_load_pm4_ucode failed %d\n", status); return status; } /* load the prefetch parser ucode */ status = kgsl_ringbuffer_load_pfp_ucode(device); if (status != 0) { KGSL_DRV_ERR("kgsl_ringbuffer_load_pfp_ucode failed %d\n", status); return status; } kgsl_yamato_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804); rb->rptr = 0; rb->wptr = 0; /* clear ME_HALT to start micro engine */ kgsl_yamato_regwrite(device, REG_CP_ME_CNTL, 0); /* ME_INIT */ cmds = kgsl_ringbuffer_allocspace(rb, 19); cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-19); GSL_RB_WRITE(cmds, cmds_gpu, PM4_HDR_ME_INIT); /* All fields present (bits 9:0) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x000003ff); /* Disable/Enable Real-Time Stream processing (present but ignored) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* Enable (2D <-> 3D) implicit synchronization (present but ignored) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL)); GSL_RB_WRITE(cmds, cmds_gpu, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE)); /* Vertex and Pixel Shader Start Addresses in instructions * (3 DWORDS per instruction) */ GSL_RB_WRITE(cmds, cmds_gpu, 0x80000180); /* Maximum Contexts */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000001); /* Write Confirm Interval and The CP will wait the * wait_interval * 16 clocks between polling */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* NQ and External Memory Swap */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* Protected mode error checking */ GSL_RB_WRITE(cmds, cmds_gpu, GSL_RB_PROTECTED_MODE_CONTROL); /* Disable header dumping and Header dump address */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); /* Header dump size */ GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); kgsl_ringbuffer_submit(rb); /* idle device to validate ME INIT */ status = kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT); KGSL_CMD_DBG("enabling CP interrupts: mask %08lx\n", GSL_CP_INT_MASK); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, GSL_CP_INT_MASK); if (status == 0) rb->flags |= KGSL_FLAGS_STARTED; KGSL_CMD_VDBG("return %d\n", status); return status; }
void kgsl_cp_intrcallback(struct kgsl_device *device) { unsigned int status = 0, num_reads = 0, master_status = 0; struct kgsl_yamato_device *yamato_device = (struct kgsl_yamato_device *) device; struct kgsl_ringbuffer *rb = &device->ringbuffer; KGSL_CMD_VDBG("enter (device=%p)\n", device); kgsl_yamato_regread(device, REG_MASTER_INT_SIGNAL, &master_status); while (!status && (num_reads < VALID_STATUS_COUNT_MAX) && (master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) { kgsl_yamato_regread(device, REG_CP_INT_STATUS, &status); kgsl_yamato_regread(device, REG_MASTER_INT_SIGNAL, &master_status); num_reads++; } if (num_reads > 1) KGSL_DRV_WARN("Looped %d times to read REG_CP_INT_STATUS\n", num_reads); if (!status) { if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) { KGSL_DRV_WARN("Unable to read CP_INT_STATUS\n"); wake_up_interruptible_all(&yamato_device->ib1_wq); } else KGSL_DRV_WARN("Spurious interrput detected\n"); return; } if (status & CP_INT_CNTL__RB_INT_MASK) { unsigned int enableflag = 0; kgsl_sharedmem_writel(&rb->device->memstore, KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), enableflag); wmb(); KGSL_CMD_WARN("ringbuffer rb interrupt\n"); } if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) { KGSL_CMD_FATAL("ringbuffer TO packet in IB interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer opcode error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer protected mode error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer reserved bit error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__IB_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer IB error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__SW_INT_MASK) KGSL_CMD_DBG("ringbuffer software interrupt\n"); if (status & CP_INT_CNTL__IB2_INT_MASK) KGSL_CMD_DBG("ringbuffer ib2 interrupt\n"); if (status & (~GSL_CP_INT_MASK)) KGSL_CMD_DBG("bad bits in REG_CP_INT_STATUS %08x\n", status); status &= GSL_CP_INT_MASK; kgsl_yamato_regwrite(device, REG_CP_INT_ACK, status); if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) { KGSL_CMD_WARN("ringbuffer ib1/rb interrupt\n"); wake_up_interruptible_all(&yamato_device->ib1_wq); atomic_notifier_call_chain(&(device->ts_notifier_list), KGSL_DEVICE_YAMATO, NULL); } KGSL_CMD_VDBG("return\n"); }
static int kgsl_ringbuffer_start(struct kgsl_ringbuffer *rb) { int status; union reg_cp_rb_cntl cp_rb_cntl; unsigned int *cmds, rb_cntl; struct kgsl_device *device = rb->device; KGSL_CMD_VDBG("enter (rb=%p)\n", rb); if (rb->flags & KGSL_FLAGS_STARTED) { KGSL_CMD_VDBG("return %d\n", 0); return 0; } kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0, sizeof(struct kgsl_rbmemptrs)); kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA, (rb->sizedwords << 2)); kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_BASE, (rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET)); kgsl_yamato_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 ); kgsl_yamato_regread(device, REG_CP_RB_CNTL, &rb_cntl); cp_rb_cntl.val = rb_cntl; cp_rb_cntl.f.rb_bufsz = kgsl_ringbuffer_sizelog2quadwords(rb->sizedwords); cp_rb_cntl.f.rb_blksz = rb->blksizequadwords; cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE; kgsl_yamato_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val); kgsl_yamato_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); kgsl_yamato_regwrite(device, REG_CP_RB_RPTR_ADDR, rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_RPTR_OFFSET); kgsl_yamato_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF); kgsl_yamato_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp)); kgsl_yamato_regwrite(device, REG_SCRATCH_UMSK, GSL_RB_MEMPTRS_SCRATCH_MASK); status = kgsl_ringbuffer_load_pm4_ucode(device); if (status != 0) { KGSL_DRV_ERR("kgsl_ringbuffer_load_pm4_ucode failed %d\n", status); return status; } status = kgsl_ringbuffer_load_pfp_ucode(device); if (status != 0) { KGSL_DRV_ERR("kgsl_ringbuffer_load_pm4_ucode failed %d\n", status); return status; } kgsl_yamato_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804); rb->rptr = 0; rb->wptr = 0; rb->timestamp = 0; GSL_RB_INIT_TIMESTAMP(rb); INIT_LIST_HEAD(&rb->memqueue); kgsl_yamato_regwrite(device, REG_CP_ME_CNTL, 0); cmds = kgsl_ringbuffer_allocspace(rb, 19); GSL_RB_WRITE(cmds, PM4_HDR_ME_INIT); GSL_RB_WRITE(cmds, 0x000003ff); GSL_RB_WRITE(cmds, 0x00000000); GSL_RB_WRITE(cmds, 0x00000000); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL)); GSL_RB_WRITE(cmds, GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE)); GSL_RB_WRITE(cmds, 0x80000180); GSL_RB_WRITE(cmds, 0x00000001); GSL_RB_WRITE(cmds, 0x00000000); GSL_RB_WRITE(cmds, 0x00000000); GSL_RB_WRITE(cmds, GSL_RB_PROTECTED_MODE_CONTROL); GSL_RB_WRITE(cmds, 0x00000000); GSL_RB_WRITE(cmds, 0x00000000); kgsl_ringbuffer_submit(rb); status = kgsl_yamato_idle(device, KGSL_TIMEOUT_DEFAULT); KGSL_CMD_DBG("enabling CP interrupts: mask %08lx\n", GSL_CP_INT_MASK); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, GSL_CP_INT_MASK); if (status == 0) rb->flags |= KGSL_FLAGS_STARTED; KGSL_CMD_VDBG("return %d\n", status); return status; }
static uint32_t kgsl_ringbuffer_addcmds(struct kgsl_ringbuffer *rb, unsigned int flags, unsigned int *cmds, int sizedwords) { unsigned int *ringcmds; unsigned int timestamp; unsigned int total_sizedwords = sizedwords + 6; /* reserve space to temporarily turn off protected mode * error checking if needed */ total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 9 : 0; ringcmds = kgsl_ringbuffer_allocspace(rb, total_sizedwords); if (flags & KGSL_CMD_FLAGS_PMODE) { /* disable protected mode error checking */ *ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1); *ringcmds++ = 0; } memcpy(ringcmds, cmds, (sizedwords << 2)); ringcmds += sizedwords; if (flags & KGSL_CMD_FLAGS_PMODE) { /* re-enable protected mode error checking */ *ringcmds++ = pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1); *ringcmds++ = 1; } rb->timestamp++; timestamp = rb->timestamp; /* start-of-pipeline and end-of-pipeline timestamps */ *ringcmds++ = pm4_type0_packet(REG_CP_TIMESTAMP, 1); *ringcmds++ = rb->timestamp; *ringcmds++ = pm4_type3_packet(PM4_EVENT_WRITE, 3); *ringcmds++ = CACHE_FLUSH_TS; *ringcmds++ = (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)); *ringcmds++ = rb->timestamp; if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) { /* Conditional execution based on memory values */ *ringcmds++ = pm4_type3_packet(PM4_COND_EXEC, 4); *ringcmds++ = (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2; *ringcmds++ = (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2; *ringcmds++ = rb->timestamp; /* # of conditional command DWORDs */ *ringcmds++ = 4; *ringcmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); *ringcmds++ = 0x00000000; *ringcmds++ = pm4_type3_packet(PM4_INTERRUPT, 1); *ringcmds++ = CP_INT_CNTL__RB_INT_MASK; }
static uint32_t adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, unsigned int flags, unsigned int *cmds, int sizedwords) { struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device); unsigned int *ringcmds; unsigned int timestamp; unsigned int total_sizedwords = sizedwords + 6; unsigned int i; unsigned int rcmd_gpu; /* reserve space to temporarily turn off protected mode * error checking if needed */ total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0; total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0; if (adreno_is_a3xx(adreno_dev)) total_sizedwords += 7; ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords); rcmd_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-total_sizedwords); if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) { GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER); } if (flags & KGSL_CMD_FLAGS_PMODE) { /* disable protected mode error checking */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_SET_PROTECTED_MODE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 0); } for (i = 0; i < sizedwords; i++) { GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds); cmds++; } if (flags & KGSL_CMD_FLAGS_PMODE) { /* re-enable protected mode error checking */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_SET_PROTECTED_MODE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 1); } rb->timestamp++; timestamp = rb->timestamp; /* start-of-pipeline and end-of-pipeline timestamps */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); if (adreno_is_a3xx(adreno_dev)) { /* * FLush HLSQ lazy updates to make sure there are no * rsources pending for indirect loads after the timestamp */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_WAIT_FOR_IDLE, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00); } GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp))); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) { /* Conditional execution based on memory values */ GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_COND_EXEC, 4)); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); /* # of conditional command DWORDs */ GSL_RB_WRITE(ringcmds, rcmd_gpu, 2); GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_INTERRUPT, 1)); GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK); }
int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram) { int status; /*cp_rb_cntl_u cp_rb_cntl; */ union reg_cp_rb_cntl cp_rb_cntl; unsigned int rb_cntl; struct kgsl_device *device = rb->device; struct adreno_device *adreno_dev = ADRENO_DEVICE(device); if (rb->flags & KGSL_FLAGS_STARTED) return 0; if (init_ram) { rb->timestamp = 0; GSL_RB_INIT_TIMESTAMP(rb); } kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0, sizeof(struct kgsl_rbmemptrs)); kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA, (rb->sizedwords << 2)); if (adreno_is_a2xx(adreno_dev)) { adreno_regwrite(device, REG_CP_RB_WPTR_BASE, (rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET)); /* setup WPTR delay */ adreno_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */); } /*setup REG_CP_RB_CNTL */ adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl); cp_rb_cntl.val = rb_cntl; /* * The size of the ringbuffer in the hardware is the log2 * representation of the size in quadwords (sizedwords / 2) */ cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1); /* * Specify the quadwords to read before updating mem RPTR. * Like above, pass the log2 representation of the blocksize * in quadwords. */ cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3); if (adreno_is_a2xx(adreno_dev)) { /* WPTR polling */ cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; } /* mem RPTR writebacks */ cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE; adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val); adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); adreno_regwrite(device, REG_CP_RB_RPTR_ADDR, rb->memptrs_desc.gpuaddr + GSL_RB_MEMPTRS_RPTR_OFFSET); if (adreno_is_a3xx(adreno_dev)) { /* enable access protection to privileged registers */ adreno_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007); /* RBBM registers */ adreno_regwrite(device, A3XX_CP_PROTECT_REG_0, 0x63000040); adreno_regwrite(device, A3XX_CP_PROTECT_REG_1, 0x62000080); adreno_regwrite(device, A3XX_CP_PROTECT_REG_2, 0x600000CC); adreno_regwrite(device, A3XX_CP_PROTECT_REG_3, 0x60000108); adreno_regwrite(device, A3XX_CP_PROTECT_REG_4, 0x64000140); adreno_regwrite(device, A3XX_CP_PROTECT_REG_5, 0x66000400); /* CP registers */ adreno_regwrite(device, A3XX_CP_PROTECT_REG_6, 0x65000700); adreno_regwrite(device, A3XX_CP_PROTECT_REG_7, 0x610007D8); adreno_regwrite(device, A3XX_CP_PROTECT_REG_8, 0x620007E0); adreno_regwrite(device, A3XX_CP_PROTECT_REG_9, 0x61001178); adreno_regwrite(device, A3XX_CP_PROTECT_REG_A, 0x64001180); /* RB registers */ adreno_regwrite(device, A3XX_CP_PROTECT_REG_B, 0x60003300); /* VBIF registers */ adreno_regwrite(device, A3XX_CP_PROTECT_REG_C, 0x6B00C000); } if (adreno_is_a2xx(adreno_dev)) { /* explicitly clear all cp interrupts */ adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF); } /* setup scratch/timestamp */ adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp)); adreno_regwrite(device, REG_SCRATCH_UMSK, GSL_RB_MEMPTRS_SCRATCH_MASK); /* load the CP ucode */ status = adreno_ringbuffer_load_pm4_ucode(device); if (status != 0) return status; /* load the prefetch parser ucode */ status = adreno_ringbuffer_load_pfp_ucode(device); if (status != 0) return status; adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804); rb->rptr = 0; rb->wptr = 0; /* clear ME_HALT to start micro engine */ adreno_regwrite(device, REG_CP_ME_CNTL, 0); /* ME init is GPU specific, so jump into the sub-function */ adreno_dev->gpudev->rb_init(adreno_dev, rb); /* idle device to validate ME INIT */ status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT); if (status == 0) rb->flags |= KGSL_FLAGS_STARTED; return status; }
/* functions */ void kgsl_cp_intrcallback(struct kgsl_device *device) { unsigned int status = 0; struct kgsl_ringbuffer *rb = &device->ringbuffer; KGSL_CMD_VDBG("enter (device=%p)\n", device); kgsl_yamato_regread(device, REG_CP_INT_STATUS, &status); if (status & CP_INT_CNTL__RB_INT_MASK) { /* signal intr completion event */ int init_reftimestamp = 0x7fffffff; int enableflag = 0; kgsl_sharedmem_write(&rb->device->memstore, KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), &enableflag, 4); kgsl_sharedmem_write(&rb->device->memstore, KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts), &init_reftimestamp, 4); KGSL_CMD_WARN("ringbuffer rb interrupt\n"); } if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) { KGSL_CMD_WARN("ringbuffer ib1/rb interrupt\n"); wake_up_interruptible_all(&device->ib1_wq); } if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) { KGSL_CMD_FATAL("ringbuffer TO packet in IB interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer opcode error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer protected mode error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer reserved bit error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__IB_ERROR_MASK) { KGSL_CMD_FATAL("ringbuffer IB error interrupt\n"); kgsl_yamato_regwrite(rb->device, REG_CP_INT_CNTL, 0); kgsl_ringbuffer_dump(rb); } if (status & CP_INT_CNTL__SW_INT_MASK) KGSL_CMD_DBG("ringbuffer software interrupt\n"); if (status & CP_INT_CNTL__IB2_INT_MASK) KGSL_CMD_DBG("ringbuffer ib2 interrupt\n"); if (status & (~GSL_CP_INT_MASK)) KGSL_CMD_DBG("bad bits in REG_CP_INT_STATUS %08x\n", status); /* only ack bits we understand */ status &= GSL_CP_INT_MASK; kgsl_yamato_regwrite(device, REG_CP_INT_ACK, status); KGSL_CMD_VDBG("return\n"); }