/******************************************************************************* * This function prepare boot argument for kernel entrypoint ******************************************************************************/ void bl31_prepare_kernel_entry(uint64_t k32_64) { entry_point_info_t *next_image_info; uint32_t image_type; /* Determine which image to execute next */ /* image_type = bl31_get_next_image_type(); */ image_type = NON_SECURE; /* Program EL3 registers to enable entry into the next EL */ if (k32_64 == 0) next_image_info = bl31_plat_get_next_kernel32_ep_info(); else next_image_info = bl31_plat_get_next_kernel64_ep_info(); assert(next_image_info); assert(image_type == GET_SECURITY_STATE(next_image_info->h.attr)); INFO("BL3-1: Preparing for EL3 exit to %s world, Kernel\n", (image_type == SECURE) ? "secure" : "normal"); INFO("BL3-1: Next image address = 0x%llx\n", (unsigned long long) next_image_info->pc); INFO("BL3-1: Next image spsr = 0x%x\n", next_image_info->spsr); cm_init_context(read_mpidr_el1(), next_image_info); cm_prepare_el3_exit(image_type); }
/******************************************************************************* * The following functions finish an earlier affinity power on request. They * are called by the common finisher routine in psci_common.c. ******************************************************************************/ static unsigned int psci_afflvl0_on_finish(aff_map_node_t *cpu_node) { unsigned int plat_state, state, rc; assert(cpu_node->level == MPIDR_AFFLVL0); /* Ensure we have been explicitly woken up by another cpu */ state = psci_get_state(cpu_node); assert(state == PSCI_STATE_ON_PENDING); /* * Plat. management: Perform the platform specific actions * for this cpu e.g. enabling the gic or zeroing the mailbox * register. The actual state of this cpu has already been * changed. */ if (psci_plat_pm_ops->affinst_on_finish) { /* Get the physical state of this cpu */ plat_state = get_phys_state(state); rc = psci_plat_pm_ops->affinst_on_finish(read_mpidr_el1(), cpu_node->level, plat_state); assert(rc == PSCI_E_SUCCESS); } /* * Arch. management: Enable data cache and manage stack memory */ psci_do_pwrup_cache_maintenance(); /* * All the platform specific actions for turning this cpu * on have completed. Perform enough arch.initialization * to run in the non-secure address space. */ bl31_arch_setup(); /* * Call the cpu on finish handler registered by the Secure Payload * Dispatcher to let it do any bookeeping. If the handler encounters an * error, it's expected to assert within */ if (psci_spd_pm && psci_spd_pm->svc_on_finish) psci_spd_pm->svc_on_finish(0); /* * Generic management: Now we just need to retrieve the * information that we had stashed away during the cpu_on * call to set this cpu on its way. */ cm_prepare_el3_exit(NON_SECURE); /* Clean caches before re-entering normal world */ dcsw_op_louis(DCCSW); rc = PSCI_E_SUCCESS; return rc; }
/******************************************************************************* * The following function finish an earlier power on request. They * are called by the common finisher routine in psci_common.c. The `state_info` * is the psci_power_state from which this CPU has woken up from. ******************************************************************************/ void psci_cpu_on_finish(unsigned int cpu_idx, psci_power_state_t *state_info) { /* * Plat. management: Perform the platform specific actions * for this cpu e.g. enabling the gic or zeroing the mailbox * register. The actual state of this cpu has already been * changed. */ psci_plat_pm_ops->pwr_domain_on_finish(state_info); #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) /* * Arch. management: Enable data cache and manage stack memory */ psci_do_pwrup_cache_maintenance(); #endif /* * All the platform specific actions for turning this cpu * on have completed. Perform enough arch.initialization * to run in the non-secure address space. */ psci_arch_setup(); /* * Lock the CPU spin lock to make sure that the context initialization * is done. Since the lock is only used in this function to create * a synchronization point with cpu_on_start(), it can be released * immediately. */ psci_spin_lock_cpu(cpu_idx); psci_spin_unlock_cpu(cpu_idx); /* Ensure we have been explicitly woken up by another cpu */ assert(psci_get_aff_info_state() == AFF_STATE_ON_PENDING); /* * Call the cpu on finish handler registered by the Secure Payload * Dispatcher to let it do any bookeeping. If the handler encounters an * error, it's expected to assert within */ if (psci_spd_pm && psci_spd_pm->svc_on_finish) psci_spd_pm->svc_on_finish(0); PUBLISH_EVENT(psci_cpu_on_finish); /* Populate the mpidr field within the cpu node array */ /* This needs to be done only once */ psci_cpu_pd_nodes[cpu_idx].mpidr = read_mpidr() & MPIDR_AFFINITY_MASK; /* * Generic management: Now we just need to retrieve the * information that we had stashed away during the cpu_on * call to set this cpu on its way. */ cm_prepare_el3_exit(NON_SECURE); }
/* * Handle SMC from a lower exception level to switch its execution state * (either from AArch64 to AArch32, or vice versa). * * smc_fid: * SMC function ID - either ARM_SIP_SVC_STATE_SWITCH_64 or * ARM_SIP_SVC_STATE_SWITCH_32. * pc_hi, pc_lo: * PC upon re-entry to the calling exception level; width dependent on the * calling exception level. * cookie_hi, cookie_lo: * Opaque pointer pairs received from the caller to pass it back, upon * re-entry. * handle: * Handle to saved context. */ int arm_execution_state_switch(unsigned int smc_fid, uint32_t pc_hi, uint32_t pc_lo, uint32_t cookie_hi, uint32_t cookie_lo, void *handle) { /* Execution state can be switched only if EL3 is AArch64 */ #ifdef AARCH64 int caller_64, from_el2, el, endianness, thumb = 0; u_register_t spsr, pc, scr, sctlr; entry_point_info_t ep; cpu_context_t *ctx = (cpu_context_t *) handle; el3_state_t *el3_ctx = get_el3state_ctx(ctx); /* That the SMC originated from NS is already validated by the caller */ /* * Disallow state switch if any of the secondaries have been brought up. */ if (psci_secondaries_brought_up()) goto exec_denied; spsr = read_ctx_reg(el3_ctx, CTX_SPSR_EL3); caller_64 = (GET_RW(spsr) == MODE_RW_64); if (caller_64) { /* * If the call originated from AArch64, expect 32-bit pointers when * switching to AArch32. */ if ((pc_hi != 0) || (cookie_hi != 0)) goto invalid_param; pc = pc_lo; /* Instruction state when entering AArch32 */ thumb = pc & 1; } else { /* Construct AArch64 PC */ pc = (((u_register_t) pc_hi) << 32) | pc_lo; } /* Make sure PC is 4-byte aligned, except for Thumb */ if ((pc & 0x3) && !thumb) goto invalid_param; /* * EL3 controls register width of the immediate lower EL only. Expect * this request from EL2/Hyp unless: * * - EL2 is not implemented; * - EL2 is implemented, but was disabled. This can be inferred from * SCR_EL3.HCE. */ from_el2 = caller_64 ? (GET_EL(spsr) == MODE_EL2) : (GET_M32(spsr) == MODE32_hyp); scr = read_ctx_reg(el3_ctx, CTX_SCR_EL3); if (!from_el2) { /* The call is from NS privilege level other than HYP */ /* * Disallow switching state if there's a Hypervisor in place; * this request must be taken up with the Hypervisor instead. */ if (scr & SCR_HCE_BIT) goto exec_denied; } /* * Return to the caller using the same endianness. Extract * endianness bit from the respective system control register * directly. */ sctlr = from_el2 ? read_sctlr_el2() : read_sctlr_el1(); endianness = !!(sctlr & SCTLR_EE_BIT); /* Construct SPSR for the exception state we're about to switch to */ if (caller_64) { int impl; /* * Switching from AArch64 to AArch32. Ensure this CPU implements * the target EL in AArch32. */ impl = from_el2 ? EL_IMPLEMENTED(2) : EL_IMPLEMENTED(1); if (impl != EL_IMPL_A64_A32) goto exec_denied; /* Return to the equivalent AArch32 privilege level */ el = from_el2 ? MODE32_hyp : MODE32_svc; spsr = SPSR_MODE32(el, thumb ? SPSR_T_THUMB : SPSR_T_ARM, endianness, DISABLE_ALL_EXCEPTIONS); } else { /* * Switching from AArch32 to AArch64. Since it's not possible to * implement an EL as AArch32-only (from which this call was * raised), it's safe to assume AArch64 is also implemented. */ el = from_el2 ? MODE_EL2 : MODE_EL1; spsr = SPSR_64(el, MODE_SP_ELX, DISABLE_ALL_EXCEPTIONS); } /* * Use the context management library to re-initialize the existing * context with the execution state flipped. Since the library takes * entry_point_info_t pointer as the argument, construct a dummy one * with PC, state width, endianness, security etc. appropriately set. * Other entries in the entry point structure are irrelevant for * purpose. */ zeromem(&ep, sizeof(ep)); ep.pc = pc; ep.spsr = spsr; SET_PARAM_HEAD(&ep, PARAM_EP, VERSION_1, ((endianness ? EP_EE_BIG : EP_EE_LITTLE) | NON_SECURE | EP_ST_DISABLE)); /* * Re-initialize the system register context, and exit EL3 as if for the * first time. State switch is effectively a soft reset of the * calling EL. */ cm_init_my_context(&ep); cm_prepare_el3_exit(NON_SECURE); /* * State switch success. The caller of SMC wouldn't see the SMC * returning. Instead, execution starts at the supplied entry point, * with context pointers populated in registers 0 and 1. */ SMC_RET2(handle, cookie_hi, cookie_lo); invalid_param: SMC_RET1(handle, STATE_SW_E_PARAM); exec_denied: #endif /* State switch denied */ SMC_RET1(handle, STATE_SW_E_DENIED); }
/******************************************************************************* * This function is responsible for handling all SMCs in the Trusted OS/App * range from the non-secure state as defined in the SMC Calling Convention * Document. It is also responsible for communicating with the Secure payload * to delegate work and return results back to the non-secure state. Lastly it * will also return any information that the secure payload needs to do the * work assigned to it. ******************************************************************************/ uint64_t tspd_smc_handler(uint32_t smc_fid, uint64_t x1, uint64_t x2, uint64_t x3, uint64_t x4, void *cookie, void *handle, uint64_t flags) { cpu_context_t *ns_cpu_context; uint32_t linear_id = plat_my_core_pos(), ns; tsp_context_t *tsp_ctx = &tspd_sp_context[linear_id]; uint64_t rc; #if TSP_INIT_ASYNC entry_point_info_t *next_image_info; #endif /* Determine which security state this SMC originated from */ ns = is_caller_non_secure(flags); switch (smc_fid) { /* * This function ID is used by TSP to indicate that it was * preempted by a normal world IRQ. * */ case TSP_PREEMPTED: if (ns) SMC_RET1(handle, SMC_UNK); return tspd_handle_sp_preemption(handle); /* * This function ID is used only by the TSP to indicate that it has * finished handling a S-EL1 FIQ interrupt. Execution should resume * in the normal world. */ case TSP_HANDLED_S_EL1_FIQ: if (ns) SMC_RET1(handle, SMC_UNK); assert(handle == cm_get_context(SECURE)); /* * Restore the relevant EL3 state which saved to service * this SMC. */ if (get_std_smc_active_flag(tsp_ctx->state)) { SMC_SET_EL3(&tsp_ctx->cpu_ctx, CTX_SPSR_EL3, tsp_ctx->saved_spsr_el3); SMC_SET_EL3(&tsp_ctx->cpu_ctx, CTX_ELR_EL3, tsp_ctx->saved_elr_el3); #if TSPD_ROUTE_IRQ_TO_EL3 /* * Need to restore the previously interrupted * secure context. */ memcpy(&tsp_ctx->cpu_ctx, &tsp_ctx->sp_ctx, TSPD_SP_CTX_SIZE); #endif } /* Get a reference to the non-secure context */ ns_cpu_context = cm_get_context(NON_SECURE); assert(ns_cpu_context); /* * Restore non-secure state. There is no need to save the * secure system register context since the TSP was supposed * to preserve it during S-EL1 interrupt handling. */ cm_el1_sysregs_context_restore(NON_SECURE); cm_set_next_eret_context(NON_SECURE); SMC_RET0((uint64_t) ns_cpu_context); /* * This function ID is used only by the TSP to indicate that it was * interrupted due to a EL3 FIQ interrupt. Execution should resume * in the normal world. */ case TSP_EL3_FIQ: if (ns) SMC_RET1(handle, SMC_UNK); assert(handle == cm_get_context(SECURE)); /* Assert that standard SMC execution has been preempted */ assert(get_std_smc_active_flag(tsp_ctx->state)); /* Save the secure system register state */ cm_el1_sysregs_context_save(SECURE); /* Get a reference to the non-secure context */ ns_cpu_context = cm_get_context(NON_SECURE); assert(ns_cpu_context); /* Restore non-secure state */ cm_el1_sysregs_context_restore(NON_SECURE); cm_set_next_eret_context(NON_SECURE); SMC_RET1(ns_cpu_context, TSP_EL3_FIQ); /* * This function ID is used only by the SP to indicate it has * finished initialising itself after a cold boot */ case TSP_ENTRY_DONE: if (ns) SMC_RET1(handle, SMC_UNK); /* * Stash the SP entry points information. This is done * only once on the primary cpu */ assert(tsp_vectors == NULL); tsp_vectors = (tsp_vectors_t *) x1; if (tsp_vectors) { set_tsp_pstate(tsp_ctx->state, TSP_PSTATE_ON); /* * TSP has been successfully initialized. Register power * managemnt hooks with PSCI */ psci_register_spd_pm_hook(&tspd_pm); /* * Register an interrupt handler for S-EL1 interrupts * when generated during code executing in the * non-secure state. */ flags = 0; set_interrupt_rm_flag(flags, NON_SECURE); rc = register_interrupt_type_handler(INTR_TYPE_S_EL1, tspd_sel1_interrupt_handler, flags); if (rc) panic(); #if TSPD_ROUTE_IRQ_TO_EL3 /* * Register an interrupt handler for NS interrupts when * generated during code executing in secure state are * routed to EL3. */ flags = 0; set_interrupt_rm_flag(flags, SECURE); rc = register_interrupt_type_handler(INTR_TYPE_NS, tspd_ns_interrupt_handler, flags); if (rc) panic(); /* * Disable the interrupt NS locally since it will be enabled globally * within cm_init_my_context. */ disable_intr_rm_local(INTR_TYPE_NS, SECURE); #endif } #if TSP_INIT_ASYNC /* Save the Secure EL1 system register context */ assert(cm_get_context(SECURE) == &tsp_ctx->cpu_ctx); cm_el1_sysregs_context_save(SECURE); /* Program EL3 registers to enable entry into the next EL */ next_image_info = bl31_plat_get_next_image_ep_info(NON_SECURE); assert(next_image_info); assert(NON_SECURE == GET_SECURITY_STATE(next_image_info->h.attr)); cm_init_my_context(next_image_info); cm_prepare_el3_exit(NON_SECURE); SMC_RET0(cm_get_context(NON_SECURE)); #else /* * SP reports completion. The SPD must have initiated * the original request through a synchronous entry * into the SP. Jump back to the original C runtime * context. */ tspd_synchronous_sp_exit(tsp_ctx, x1); #endif /* * These function IDs is used only by the SP to indicate it has * finished: * 1. turning itself on in response to an earlier psci * cpu_on request * 2. resuming itself after an earlier psci cpu_suspend * request. */ case TSP_ON_DONE: case TSP_RESUME_DONE: /* * These function IDs is used only by the SP to indicate it has * finished: * 1. suspending itself after an earlier psci cpu_suspend * request. * 2. turning itself off in response to an earlier psci * cpu_off request. */ case TSP_OFF_DONE: case TSP_SUSPEND_DONE: case TSP_SYSTEM_OFF_DONE: case TSP_SYSTEM_RESET_DONE: if (ns) SMC_RET1(handle, SMC_UNK); /* * SP reports completion. The SPD must have initiated the * original request through a synchronous entry into the SP. * Jump back to the original C runtime context, and pass x1 as * return value to the caller */ tspd_synchronous_sp_exit(tsp_ctx, x1); /* * Request from non-secure client to perform an * arithmetic operation or response from secure * payload to an earlier request. */ case TSP_FAST_FID(TSP_ADD): case TSP_FAST_FID(TSP_SUB): case TSP_FAST_FID(TSP_MUL): case TSP_FAST_FID(TSP_DIV): case TSP_STD_FID(TSP_ADD): case TSP_STD_FID(TSP_SUB): case TSP_STD_FID(TSP_MUL): case TSP_STD_FID(TSP_DIV): if (ns) { /* * This is a fresh request from the non-secure client. * The parameters are in x1 and x2. Figure out which * registers need to be preserved, save the non-secure * state and send the request to the secure payload. */ assert(handle == cm_get_context(NON_SECURE)); /* Check if we are already preempted */ if (get_std_smc_active_flag(tsp_ctx->state)) SMC_RET1(handle, SMC_UNK); cm_el1_sysregs_context_save(NON_SECURE); /* Save x1 and x2 for use by TSP_GET_ARGS call below */ store_tsp_args(tsp_ctx, x1, x2); /* * We are done stashing the non-secure context. Ask the * secure payload to do the work now. */ /* * Verify if there is a valid context to use, copy the * operation type and parameters to the secure context * and jump to the fast smc entry point in the secure * payload. Entry into S-EL1 will take place upon exit * from this function. */ assert(&tsp_ctx->cpu_ctx == cm_get_context(SECURE)); /* Set appropriate entry for SMC. * We expect the TSP to manage the PSTATE.I and PSTATE.F * flags as appropriate. */ if (GET_SMC_TYPE(smc_fid) == SMC_TYPE_FAST) { cm_set_elr_el3(SECURE, (uint64_t) &tsp_vectors->fast_smc_entry); } else { set_std_smc_active_flag(tsp_ctx->state); cm_set_elr_el3(SECURE, (uint64_t) &tsp_vectors->std_smc_entry); #if TSPD_ROUTE_IRQ_TO_EL3 /* * Enable the routing of NS interrupts to EL3 * during STD SMC processing on this core. */ enable_intr_rm_local(INTR_TYPE_NS, SECURE); #endif } cm_el1_sysregs_context_restore(SECURE); cm_set_next_eret_context(SECURE); SMC_RET3(&tsp_ctx->cpu_ctx, smc_fid, x1, x2); } else { /* * This is the result from the secure client of an * earlier request. The results are in x1-x3. Copy it * into the non-secure context, save the secure state * and return to the non-secure state. */ assert(handle == cm_get_context(SECURE)); cm_el1_sysregs_context_save(SECURE); /* Get a reference to the non-secure context */ ns_cpu_context = cm_get_context(NON_SECURE); assert(ns_cpu_context); /* Restore non-secure state */ cm_el1_sysregs_context_restore(NON_SECURE); cm_set_next_eret_context(NON_SECURE); if (GET_SMC_TYPE(smc_fid) == SMC_TYPE_STD) { clr_std_smc_active_flag(tsp_ctx->state); #if TSPD_ROUTE_IRQ_TO_EL3 /* * Disable the routing of NS interrupts to EL3 * after STD SMC processing is finished on this * core. */ disable_intr_rm_local(INTR_TYPE_NS, SECURE); #endif } SMC_RET3(ns_cpu_context, x1, x2, x3); } break; /* * Request from non secure world to resume the preempted * Standard SMC call. */ case TSP_FID_RESUME: /* RESUME should be invoked only by normal world */ if (!ns) { assert(0); break; } /* * This is a resume request from the non-secure client. * save the non-secure state and send the request to * the secure payload. */ assert(handle == cm_get_context(NON_SECURE)); /* Check if we are already preempted before resume */ if (!get_std_smc_active_flag(tsp_ctx->state)) SMC_RET1(handle, SMC_UNK); cm_el1_sysregs_context_save(NON_SECURE); /* * We are done stashing the non-secure context. Ask the * secure payload to do the work now. */ #if TSPD_ROUTE_IRQ_TO_EL3 /* * Enable the routing of NS interrupts to EL3 during resumption * of STD SMC call on this core. */ enable_intr_rm_local(INTR_TYPE_NS, SECURE); #endif /* We just need to return to the preempted point in * TSP and the execution will resume as normal. */ cm_el1_sysregs_context_restore(SECURE); cm_set_next_eret_context(SECURE); SMC_RET0(&tsp_ctx->cpu_ctx); /* * This is a request from the secure payload for more arguments * for an ongoing arithmetic operation requested by the * non-secure world. Simply return the arguments from the non- * secure client in the original call. */ case TSP_GET_ARGS: if (ns) SMC_RET1(handle, SMC_UNK); get_tsp_args(tsp_ctx, x1, x2); SMC_RET2(handle, x1, x2); case TOS_CALL_COUNT: /* * Return the number of service function IDs implemented to * provide service to non-secure */ SMC_RET1(handle, TSP_NUM_FID); case TOS_UID: /* Return TSP UID to the caller */ SMC_UUID_RET(handle, tsp_uuid); case TOS_CALL_VERSION: /* Return the version of current implementation */ SMC_RET2(handle, TSP_VERSION_MAJOR, TSP_VERSION_MINOR); default: break; } SMC_RET1(handle, SMC_UNK); }
/******************************************************************************* * The following functions finish an earlier affinity suspend request. They * are called by the common finisher routine in psci_common.c. ******************************************************************************/ static unsigned int psci_afflvl0_suspend_finish(aff_map_node_t *cpu_node) { unsigned int plat_state, state, rc; int32_t suspend_level; uint64_t counter_freq; assert(cpu_node->level == MPIDR_AFFLVL0); /* Ensure we have been woken up from a suspended state */ state = psci_get_state(cpu_node); assert(state == PSCI_STATE_SUSPEND); /* * Plat. management: Perform the platform specific actions * before we change the state of the cpu e.g. enabling the * gic or zeroing the mailbox register. If anything goes * wrong then assert as there is no way to recover from this * situation. */ if (psci_plat_pm_ops->affinst_suspend_finish) { /* Get the physical state of this cpu */ plat_state = get_phys_state(state); rc = psci_plat_pm_ops->affinst_suspend_finish(read_mpidr_el1(), cpu_node->level, plat_state); assert(rc == PSCI_E_SUCCESS); } /* Get the index for restoring the re-entry information */ /* * Arch. management: Enable the data cache, manage stack memory and * restore the stashed EL3 architectural context from the 'cpu_context' * structure for this cpu. */ psci_do_pwrup_cache_maintenance(); /* Re-init the cntfrq_el0 register */ counter_freq = plat_get_syscnt_freq(); write_cntfrq_el0(counter_freq); /* * Call the cpu suspend finish handler registered by the Secure Payload * Dispatcher to let it do any bookeeping. If the handler encounters an * error, it's expected to assert within */ if (psci_spd_pm && psci_spd_pm->svc_suspend) { suspend_level = psci_get_suspend_afflvl(); assert (suspend_level != PSCI_INVALID_DATA); psci_spd_pm->svc_suspend_finish(suspend_level); } /* Invalidate the suspend context for the node */ psci_set_suspend_power_state(PSCI_INVALID_DATA); /* * Generic management: Now we just need to retrieve the * information that we had stashed away during the suspend * call to set this cpu on its way. */ cm_prepare_el3_exit(NON_SECURE); /* Clean caches before re-entering normal world */ dcsw_op_louis(DCCSW); rc = PSCI_E_SUCCESS; return rc; }
/****************************************************************************** * PSCI Library interface to initialize the cpu context for the next non * secure image during cold boot. The relevant registers in the cpu context * need to be retrieved and programmed on return from this interface. *****************************************************************************/ void psci_prepare_next_non_secure_ctx(entry_point_info_t *next_image_info) { assert(GET_SECURITY_STATE(next_image_info->h.attr) == NON_SECURE); cm_init_my_context(next_image_info); cm_prepare_el3_exit(NON_SECURE); }