void
ipcCancel(tcb_t *tptr)
{
thread_state_t *state = &tptr->tcbState;
switch (thread_state_ptr_get_tsType(state)) {
case ThreadState_BlockedOnSend:
case ThreadState_BlockedOnReceive: {
/* blockedIPCCancel state */
endpoint_t *epptr;
tcb_queue_t queue;
epptr = EP_PTR(thread_state_ptr_get_blockingIPCEndpoint(state));
/* Haskell error "blockedIPCCancel: endpoint must not be idle" */
assert(endpoint_ptr_get_state(epptr) != EPState_Idle);
/* Dequeue TCB */
queue = ep_ptr_get_queue(epptr);
queue = tcbEPDequeue(tptr, queue);
ep_ptr_set_queue(epptr, queue);
if (!queue.head) {
endpoint_ptr_set_state(epptr, EPState_Idle);
}
setThreadState(tptr, ThreadState_Inactive);
break;
}
case ThreadState_BlockedOnAsyncEvent:
asyncIPCCancel(tptr,
AEP_PTR(thread_state_ptr_get_blockingIPCEndpoint(state)));
break;
case ThreadState_BlockedOnReply: {
cte_t *slot, *callerCap;
fault_null_fault_ptr_new(&tptr->tcbFault);
/* Get the reply cap slot */
slot = TCB_PTR_CTE_PTR(tptr, tcbReply);
callerCap = CTE_PTR(cap_reply_cap_get_capCallerSlot(slot->cap));
if (callerCap) {
finaliseCap(callerCap->cap, true, true);
callerCap->cap = cap_null_cap_new();
}
cap_reply_cap_ptr_set_capCallerSlot(&slot->cap, CTE_REF(NULL));
break;
}
}
}
void
setupCallerCap(tcb_t *sender, tcb_t *receiver)
{
cte_t *replySlot, *callerSlot;
cap_t masterCap UNUSED, callerCap UNUSED;
setThreadState(sender, ThreadState_BlockedOnReply);
replySlot = TCB_PTR_CTE_PTR(sender, tcbReply);
callerSlot = TCB_PTR_CTE_PTR(receiver, tcbCaller);
masterCap = replySlot->cap;
/* Haskell error: "Sender must have a valid master reply cap" */
assert(cap_get_capType(masterCap) == cap_reply_cap);
assert(cap_reply_cap_get_capReplyMaster(masterCap));
assert(TCB_PTR(cap_reply_cap_get_capTCBPtr(masterCap)) == NULL);
cap_reply_cap_ptr_set_capCallerSlot(&replySlot->cap, CTE_REF(callerSlot));
callerCap = callerSlot->cap;
/* Haskell error: "Caller cap must not already exist" */
assert(cap_get_capType(callerCap) == cap_null_cap);
callerSlot->cap = cap_reply_cap_new(CTE_REF(NULL), false, TCB_REF(sender));
}
void
#ifdef ARCH_X86
NORETURN
#endif
fastpath_call(word_t cptr, word_t msgInfo)
{
seL4_MessageInfo_t info;
cap_t ep_cap;
endpoint_t *ep_ptr;
word_t length;
tcb_t *dest;
word_t badge;
cte_t *replySlot, *callerSlot;
cap_t newVTable;
vspace_root_t *cap_pd;
pde_t stored_hw_asid;
word_t fault_type;
dom_t dom;
word_t replyCanGrant;
/* Get message info, length, and fault type. */
info = messageInfoFromWord_raw(msgInfo);
length = seL4_MessageInfo_get_length(info);
fault_type = seL4_Fault_get_seL4_FaultType(NODE_STATE(ksCurThread)->tcbFault);
/* Check there's no extra caps, the length is ok and there's no
* saved fault. */
if (unlikely(fastpath_mi_check(msgInfo) ||
fault_type != seL4_Fault_NullFault)) {
slowpath(SysCall);
}
/* Lookup the cap */
ep_cap = lookup_fp(TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCTable)->cap, cptr);
/* Check it's an endpoint */
if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) ||
!cap_endpoint_cap_get_capCanSend(ep_cap))) {
slowpath(SysCall);
}
/* Get the endpoint address */
ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap));
/* Get the destination thread, which is only going to be valid
* if the endpoint is valid. */
dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr));
/* Check that there's a thread waiting to receive */
if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) {
slowpath(SysCall);
}
/* ensure we are not single stepping the destination in ia32 */
#if defined(CONFIG_HARDWARE_DEBUG_API) && defined(CONFIG_ARCH_IA32)
if (dest->tcbArch.tcbContext.breakpointState.single_step_enabled) {
slowpath(SysCall);
}
#endif
/* Get destination thread.*/
newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap;
/* Get vspace root. */
cap_pd = cap_vtable_cap_get_vspace_root_fp(newVTable);
/* Ensure that the destination has a valid VTable. */
if (unlikely(! isValidVTableRoot_fp(newVTable))) {
slowpath(SysCall);
}
#ifdef CONFIG_ARCH_AARCH32
/* Get HW ASID */
stored_hw_asid = cap_pd[PD_ASID_SLOT];
#endif
#ifdef CONFIG_ARCH_X86_64
/* borrow the stored_hw_asid for PCID */
stored_hw_asid.words[0] = cap_pml4_cap_get_capPML4MappedASID_fp(newVTable);
#endif
#ifdef CONFIG_ARCH_AARCH64
stored_hw_asid.words[0] = cap_page_global_directory_cap_get_capPGDMappedASID(newVTable);
#endif
#ifdef CONFIG_ARCH_RISCV
/* Get HW ASID */
stored_hw_asid.words[0] = cap_page_table_cap_get_capPTMappedASID(newVTable);
#endif
/* let gcc optimise this out for 1 domain */
dom = maxDom ? ksCurDomain : 0;
/* ensure only the idle thread or lower prio threads are present in the scheduler */
if (likely(dest->tcbPriority < NODE_STATE(ksCurThread->tcbPriority)) &&
!isHighestPrio(dom, dest->tcbPriority)) {
slowpath(SysCall);
}
/* Ensure that the endpoint has has grant or grant-reply rights so that we can
* create the reply cap */
if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap) &&
!cap_endpoint_cap_get_capCanGrantReply(ep_cap))) {
slowpath(SysCall);
}
#ifdef CONFIG_ARCH_AARCH32
if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) {
slowpath(SysCall);
}
#endif
/* Ensure the original caller is in the current domain and can be scheduled directly. */
if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) {
slowpath(SysCall);
}
#ifdef ENABLE_SMP_SUPPORT
/* Ensure both threads have the same affinity */
if (unlikely(NODE_STATE(ksCurThread)->tcbAffinity != dest->tcbAffinity)) {
slowpath(SysCall);
}
#endif /* ENABLE_SMP_SUPPORT */
/*
* --- POINT OF NO RETURN ---
*
* At this stage, we have committed to performing the IPC.
*/
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
ksKernelEntry.is_fastpath = true;
#endif
/* Dequeue the destination. */
endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext));
if (unlikely(dest->tcbEPNext)) {
dest->tcbEPNext->tcbEPPrev = NULL;
} else {
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle);
}
badge = cap_endpoint_cap_get_capEPBadge(ep_cap);
/* Block sender */
thread_state_ptr_set_tsType_np(&NODE_STATE(ksCurThread)->tcbState,
ThreadState_BlockedOnReply);
/* Get sender reply slot */
replySlot = TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbReply);
/* Get dest caller slot */
callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller);
/* Insert reply cap */
replyCanGrant = thread_state_ptr_get_blockingIPCCanGrant(&dest->tcbState);;
cap_reply_cap_ptr_new_np(&callerSlot->cap, replyCanGrant, 0,
TCB_REF(NODE_STATE(ksCurThread)));
mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot));
mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
&replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1);
fastpath_copy_mrs(length, NODE_STATE(ksCurThread), dest);
/* Dest thread is set Running, but not queued. */
thread_state_ptr_set_tsType_np(&dest->tcbState,
ThreadState_Running);
switchToThread_fp(dest, cap_pd, stored_hw_asid);
msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0));
fastpath_restore(badge, msgInfo, NODE_STATE(ksCurThread));
}
void
#ifdef ARCH_X86
NORETURN
#endif
fastpath_call(word_t cptr, word_t msgInfo)
{
seL4_MessageInfo_t info;
cap_t ep_cap;
endpoint_t *ep_ptr;
word_t length;
tcb_t *dest;
word_t badge;
cte_t *replySlot, *callerSlot;
cap_t newVTable;
pde_t *cap_pd;
pde_t stored_hw_asid;
word_t fault_type;
/* Get message info, length, and fault type. */
info = messageInfoFromWord_raw(msgInfo);
length = seL4_MessageInfo_get_length(info);
fault_type = fault_get_faultType(ksCurThread->tcbFault);
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
ksKernelEntry.path = Entry_Syscall;
ksKernelEntry.syscall_no = SysCall;
ksKernelEntry.cap_type = cap_endpoint_cap;
ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info);
ksKernelEntry.is_fastpath = true;
benchmark_track_start();
#endif
#ifdef CONFIG_BENCHMARK_TRACK_UTILISATION
benchmark_utilisation_kentry_stamp();
#endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */
/* Check there's no extra caps, the length is ok and there's no
* saved fault. */
if (unlikely(fastpath_mi_check(msgInfo) ||
fault_type != fault_null_fault)) {
slowpath(SysCall);
}
/* Lookup the cap */
ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr);
/* Check it's an endpoint */
if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) ||
!cap_endpoint_cap_get_capCanSend(ep_cap))) {
slowpath(SysCall);
}
/* Get the endpoint address */
ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap));
/* Get the destination thread, which is only going to be valid
* if the endpoint is valid. */
dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr));
/* Check that there's a thread waiting to receive */
if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) {
slowpath(SysCall);
}
/* Get destination thread.*/
newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap;
/* Get vspace root. */
#if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING)
cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable));
#else
cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable));
#endif
/* Ensure that the destination has a valid VTable. */
if (unlikely(! isValidVTableRoot_fp(newVTable))) {
slowpath(SysCall);
}
#ifdef ARCH_ARM
/* Get HW ASID */
stored_hw_asid = cap_pd[PD_ASID_SLOT];
#endif
/* Ensure the destination has a higher/equal priority to us. */
if (unlikely(dest->tcbPriority < ksCurThread->tcbPriority)) {
slowpath(SysCall);
}
/* Ensure that the endpoint has has grant rights so that we can
* create the reply cap */
if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap))) {
slowpath(SysCall);
}
#ifdef ARCH_ARM
if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) {
slowpath(SysCall);
}
#endif
/* Ensure the original caller is in the current domain and can be scheduled directly. */
if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) {
slowpath(SysCall);
}
/*
* --- POINT OF NO RETURN ---
*
* At this stage, we have committed to performing the IPC.
*/
#ifdef ARCH_X86
/* Need to update NextIP in the calling thread */
setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2);
#endif
/* Dequeue the destination. */
endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext));
if (unlikely(dest->tcbEPNext)) {
dest->tcbEPNext->tcbEPPrev = NULL;
} else {
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle);
}
badge = cap_endpoint_cap_get_capEPBadge(ep_cap);
/* Block sender */
thread_state_ptr_set_tsType_np(&ksCurThread->tcbState,
ThreadState_BlockedOnReply);
/* Get sender reply slot */
replySlot = TCB_PTR_CTE_PTR(ksCurThread, tcbReply);
/* Get dest caller slot */
callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller);
/* Insert reply cap */
cap_reply_cap_ptr_new_np(&callerSlot->cap, 0, TCB_REF(ksCurThread));
mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot));
mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
&replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1);
fastpath_copy_mrs (length, ksCurThread, dest);
/* Dest thread is set Running, but not queued. */
thread_state_ptr_set_tsType_np(&dest->tcbState,
ThreadState_Running);
switchToThread_fp(dest, cap_pd, stored_hw_asid);
msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0));
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
benchmark_track_exit();
#endif
fastpath_restore(badge, msgInfo, ksCurThread);
}