exception_t decodeSetEPTRoot(cap_t cap, extra_caps_t extraCaps) { tcb_t *tcb; cte_t *rootSlot; exception_t e; if (extraCaps.excaprefs[0] == NULL) { userError("TCB SetEPTRoot: Truncated message."); current_syscall_error.type = seL4_TruncatedMessage; return EXCEPTION_SYSCALL_ERROR; } if (cap_get_capType(extraCaps.excaprefs[0]->cap) != cap_ept_page_directory_pointer_table_cap) { userError("TCB SetEPTRoot: EPT PDPT is invalid."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } tcb = TCB_PTR(cap_thread_cap_get_capTCBPtr(cap)); rootSlot = TCB_PTR_CTE_PTR(tcb, tcbArchEPTRoot); e = cteDelete(rootSlot, true); if (e != EXCEPTION_NONE) { return e; } cteInsert(extraCaps.excaprefs[0]->cap, extraCaps.excaprefs[0], rootSlot); setThreadState(ksCurThread, ThreadState_Restart); return EXCEPTION_NONE; }
void doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot) { assert(thread_state_get_tsType(receiver->tcbState) == ThreadState_BlockedOnReply); if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) { doIPCTransfer(sender, NULL, 0, true, receiver, false); setThreadState(receiver, ThreadState_Running); attemptSwitchTo(receiver); } else { bool_t restart; restart = handleFaultReply(receiver, sender); fault_null_fault_ptr_new(&receiver->tcbFault); if (restart) { setThreadState(receiver, ThreadState_Restart); attemptSwitchTo(receiver); } else { setThreadState(receiver, ThreadState_Inactive); } } if (cap_reply_cap_get_capInCDT(slot->cap)) { cte_t *replySlot = TCB_PTR_CTE_PTR(receiver, tcbReply); assert(cap_get_capType(replySlot->cap) == cap_reply_cap); assert(cap_reply_cap_get_capInCDT(replySlot->cap)); cdtRemove(replySlot); cdtRemove(slot); slot->cap = cap_null_cap_new(); replySlot->cap = cap_reply_cap_new(false, true, TCB_REF(NULL)); } else { deleteCallerCap(sender); } }
static void handleReply(void) { cte_t *callerSlot; cap_t callerCap; callerSlot = TCB_PTR_CTE_PTR(ksCurThread, tcbCaller); callerCap = callerSlot->cap; switch (cap_get_capType(callerCap)) { case cap_reply_cap: { tcb_t *caller; if (cap_reply_cap_get_capReplyMaster(callerCap)) { break; } caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* Haskell error: * "handleReply: caller must not be the current thread" */ assert(caller != ksCurThread); doReplyTransfer(ksCurThread, caller, callerSlot); return; } case cap_null_cap: userError("Attempted reply operation when no reply cap present."); return; default: break; } fail("handleReply: invalid caller cap"); }
word_t* PURE lookupIPCBuffer(bool_t isReceiver, tcb_t *thread) { word_t w_bufferPtr; cap_t bufferCap; vm_rights_t vm_rights; w_bufferPtr = thread->tcbIPCBuffer; bufferCap = TCB_PTR_CTE_PTR(thread, tcbBuffer)->cap; if (cap_get_capType(bufferCap) != cap_frame_cap) { return NULL; } vm_rights = cap_frame_cap_get_capFVMRights(bufferCap); if (vm_rights == VMReadWrite || (!isReceiver && vm_rights == VMReadOnly)) { word_t basePtr; unsigned int pageBits; basePtr = cap_frame_cap_get_capFBasePtr(bufferCap); pageBits = pageBitsForSize(cap_frame_cap_get_capFSize(bufferCap)); return (word_t *)(basePtr + (w_bufferPtr & MASK(pageBits))); } else { return NULL; } }
void deleteCallerCap(tcb_t *receiver) { cte_t *callerSlot; callerSlot = TCB_PTR_CTE_PTR(receiver, tcbCaller); cteDeleteOne(callerSlot); }
void ipcCancel(tcb_t *tptr) { thread_state_t *state = &tptr->tcbState; switch (thread_state_ptr_get_tsType(state)) { case ThreadState_BlockedOnSend: case ThreadState_BlockedOnReceive: { /* blockedIPCCancel state */ endpoint_t *epptr; tcb_queue_t queue; epptr = EP_PTR(thread_state_ptr_get_blockingIPCEndpoint(state)); /* Haskell error "blockedIPCCancel: endpoint must not be idle" */ assert(endpoint_ptr_get_state(epptr) != EPState_Idle); /* Dequeue TCB */ queue = ep_ptr_get_queue(epptr); queue = tcbEPDequeue(tptr, queue); ep_ptr_set_queue(epptr, queue); if (!queue.head) { endpoint_ptr_set_state(epptr, EPState_Idle); } setThreadState(tptr, ThreadState_Inactive); break; } case ThreadState_BlockedOnAsyncEvent: asyncIPCCancel(tptr, AEP_PTR(thread_state_ptr_get_blockingIPCEndpoint(state))); break; case ThreadState_BlockedOnReply: { cte_t *slot, *callerCap; fault_null_fault_ptr_new(&tptr->tcbFault); /* Get the reply cap slot */ slot = TCB_PTR_CTE_PTR(tptr, tcbReply); callerCap = CTE_PTR(cap_reply_cap_get_capCallerSlot(slot->cap)); if (callerCap) { finaliseCap(callerCap->cap, true, true); callerCap->cap = cap_null_cap_new(); } cap_reply_cap_ptr_set_capCallerSlot(&slot->cap, CTE_REF(NULL)); break; } } }
void cancelIPC(tcb_t *tptr) { thread_state_t *state = &tptr->tcbState; switch (thread_state_ptr_get_tsType(state)) { case ThreadState_BlockedOnSend: case ThreadState_BlockedOnReceive: { /* blockedIPCCancel state */ endpoint_t *epptr; tcb_queue_t queue; epptr = EP_PTR(thread_state_ptr_get_blockingObject(state)); /* Haskell error "blockedIPCCancel: endpoint must not be idle" */ assert(endpoint_ptr_get_state(epptr) != EPState_Idle); /* Dequeue TCB */ queue = ep_ptr_get_queue(epptr); queue = tcbEPDequeue(tptr, queue); ep_ptr_set_queue(epptr, queue); if (!queue.head) { endpoint_ptr_set_state(epptr, EPState_Idle); } setThreadState(tptr, ThreadState_Inactive); break; } case ThreadState_BlockedOnNotification: cancelSignal(tptr, NTFN_PTR(thread_state_ptr_get_blockingObject(state))); break; case ThreadState_BlockedOnReply: { cte_t *slot, *callerCap; tptr->tcbFault = seL4_Fault_NullFault_new(); /* Get the reply cap slot */ slot = TCB_PTR_CTE_PTR(tptr, tcbReply); callerCap = CTE_PTR(mdb_node_get_mdbNext(slot->cteMDBNode)); if (callerCap) { /** GHOSTUPD: "(True, gs_set_assn cteDeleteOne_'proc (ucast cap_reply_cap))" */ cteDeleteOne(callerCap); } break; } } }
void setupCallerCap(tcb_t *sender, tcb_t *receiver) { cte_t *replySlot, *callerSlot; cap_t masterCap UNUSED, callerCap UNUSED; setThreadState(sender, ThreadState_BlockedOnReply); replySlot = TCB_PTR_CTE_PTR(sender, tcbReply); callerSlot = TCB_PTR_CTE_PTR(receiver, tcbCaller); masterCap = replySlot->cap; /* Haskell error: "Sender must have a valid master reply cap" */ assert(cap_get_capType(masterCap) == cap_reply_cap); assert(cap_reply_cap_get_capReplyMaster(masterCap)); assert(TCB_PTR(cap_reply_cap_get_capTCBPtr(masterCap)) == NULL); cap_reply_cap_ptr_set_capCallerSlot(&replySlot->cap, CTE_REF(callerSlot)); callerCap = callerSlot->cap; /* Haskell error: "Caller cap must not already exist" */ assert(cap_get_capType(callerCap) == cap_null_cap); callerSlot->cap = cap_reply_cap_new(CTE_REF(NULL), false, TCB_REF(sender)); }
void deleteCallerCap(tcb_t *receiver) { cte_t *callerSlot; callerSlot = TCB_PTR_CTE_PTR(receiver, tcbCaller); if (cap_get_capType(callerSlot->cap) == cap_reply_cap) { finaliseCap(callerSlot->cap, true, true); callerSlot->cap = cap_null_cap_new(); } }
static void handleReply(void) { cte_t *callerSlot; cap_t callerCap; callerSlot = TCB_PTR_CTE_PTR(ksCurThread, tcbCaller); callerCap = callerSlot->cap; #if defined(DEBUG) || defined(CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES) ksKernelEntry.cap_type = cap_get_capType(callerCap); #endif switch (cap_get_capType(callerCap)) { case cap_reply_cap: { tcb_t *caller; if (cap_reply_cap_get_capReplyMaster(callerCap)) { break; } caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* Haskell error: * "handleReply: caller must not be the current thread" */ assert(caller != ksCurThread); doReplyTransfer(ksCurThread, caller, callerSlot); return; } case cap_null_cap: userError("Attempted reply operation when no reply cap present."); return; default: break; } fail("handleReply: invalid caller cap"); }
void fastpath_reply_recv(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; cte_t *callerSlot; cap_t callerCap; tcb_t *caller; word_t badge; tcb_t *endpointTail; word_t fault_type; cap_t newVTable; vspace_root_t *cap_pd; pde_t stored_hw_asid; dom_t dom; /* Get message info and length */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = seL4_Fault_get_seL4_FaultType(NODE_STATE(ksCurThread)->tcbFault); /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != seL4_Fault_NullFault)) { slowpath(SysReplyRecv); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanReceive(ep_cap))) { slowpath(SysReplyRecv); } /* Check there is nothing waiting on the notification */ if (NODE_STATE(ksCurThread)->tcbBoundNotification && notification_ptr_get_state(NODE_STATE(ksCurThread)->tcbBoundNotification) == NtfnState_Active) { slowpath(SysReplyRecv); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Check that there's not a thread waiting to send */ if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) { slowpath(SysReplyRecv); } /* Only reply if the reply cap is valid. */ callerSlot = TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCaller); callerCap = callerSlot->cap; if (unlikely(!fastpath_reply_cap_check(callerCap))) { slowpath(SysReplyRecv); } /* Determine who the caller is. */ caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* ensure we are not single stepping the caller in ia32 */ #if defined(CONFIG_HARDWARE_DEBUG_API) && defined(CONFIG_ARCH_IA32) if (caller->tcbArch.tcbContext.breakpointState.single_step_enabled) { slowpath(SysReplyRecv); } #endif /* Check that the caller has not faulted, in which case a fault reply is generated instead. */ fault_type = seL4_Fault_get_seL4_FaultType(caller->tcbFault); if (unlikely(fault_type != seL4_Fault_NullFault)) { slowpath(SysReplyRecv); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap; /* Get vspace root. */ cap_pd = cap_vtable_cap_get_vspace_root_fp(newVTable); /* Ensure that the destination has a valid MMU. */ if (unlikely(! isValidVTableRoot_fp(newVTable))) { slowpath(SysReplyRecv); } #ifdef CONFIG_ARCH_AARCH32 /* Get HWASID. */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif #ifdef CONFIG_ARCH_X86_64 stored_hw_asid.words[0] = cap_pml4_cap_get_capPML4MappedASID(newVTable); #endif #ifdef CONFIG_ARCH_AARCH64 stored_hw_asid.words[0] = cap_page_global_directory_cap_get_capPGDMappedASID(newVTable); #endif #ifdef CONFIG_ARCH_RISCV stored_hw_asid.words[0] = cap_page_table_cap_get_capPTMappedASID(newVTable); #endif /* Ensure the original caller can be scheduled directly. */ dom = maxDom ? ksCurDomain : 0; if (unlikely(!isHighestPrio(dom, caller->tcbPriority))) { slowpath(SysReplyRecv); } #ifdef CONFIG_ARCH_AARCH32 /* Ensure the HWASID is valid. */ if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysReplyRecv); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysReplyRecv); } #ifdef ENABLE_SMP_SUPPORT /* Ensure both threads have the same affinity */ if (unlikely(NODE_STATE(ksCurThread)->tcbAffinity != caller->tcbAffinity)) { slowpath(SysReplyRecv); } #endif /* ENABLE_SMP_SUPPORT */ /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.is_fastpath = true; #endif /* Set thread state to BlockedOnReceive */ thread_state_ptr_mset_blockingObject_tsType( &NODE_STATE(ksCurThread)->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive); thread_state_ptr_set_blockingIPCCanGrant(&NODE_STATE(ksCurThread)->tcbState, cap_endpoint_cap_get_capCanGrant(ep_cap));; /* Place the thread in the endpoint queue */ endpointTail = endpoint_ptr_get_epQueue_tail_fp(ep_ptr); if (likely(!endpointTail)) { NODE_STATE(ksCurThread)->tcbEPPrev = NULL; NODE_STATE(ksCurThread)->tcbEPNext = NULL; /* Set head/tail of queue and endpoint state. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(NODE_STATE(ksCurThread))); endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)), EPState_Recv); } else { /* Append current thread onto the queue. */ endpointTail->tcbEPNext = NODE_STATE(ksCurThread); NODE_STATE(ksCurThread)->tcbEPPrev = endpointTail; NODE_STATE(ksCurThread)->tcbEPNext = NULL; /* Update tail of queue. */ endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)), EPState_Recv); } /* Delete the reply cap. */ mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode, 0, 1, 1); callerSlot->cap = cap_null_cap_new(); callerSlot->cteMDBNode = nullMDBNode; /* I know there's no fault, so straight to the transfer. */ /* Replies don't have a badge. */ badge = 0; fastpath_copy_mrs(length, NODE_STATE(ksCurThread), caller); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&caller->tcbState, ThreadState_Running); switchToThread_fp(caller, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); fastpath_restore(badge, msgInfo, NODE_STATE(ksCurThread)); }
void #ifdef ARCH_X86 NORETURN #endif fastpath_call(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; tcb_t *dest; word_t badge; cte_t *replySlot, *callerSlot; cap_t newVTable; vspace_root_t *cap_pd; pde_t stored_hw_asid; word_t fault_type; dom_t dom; word_t replyCanGrant; /* Get message info, length, and fault type. */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = seL4_Fault_get_seL4_FaultType(NODE_STATE(ksCurThread)->tcbFault); /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != seL4_Fault_NullFault)) { slowpath(SysCall); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanSend(ep_cap))) { slowpath(SysCall); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Get the destination thread, which is only going to be valid * if the endpoint is valid. */ dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr)); /* Check that there's a thread waiting to receive */ if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) { slowpath(SysCall); } /* ensure we are not single stepping the destination in ia32 */ #if defined(CONFIG_HARDWARE_DEBUG_API) && defined(CONFIG_ARCH_IA32) if (dest->tcbArch.tcbContext.breakpointState.single_step_enabled) { slowpath(SysCall); } #endif /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap; /* Get vspace root. */ cap_pd = cap_vtable_cap_get_vspace_root_fp(newVTable); /* Ensure that the destination has a valid VTable. */ if (unlikely(! isValidVTableRoot_fp(newVTable))) { slowpath(SysCall); } #ifdef CONFIG_ARCH_AARCH32 /* Get HW ASID */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif #ifdef CONFIG_ARCH_X86_64 /* borrow the stored_hw_asid for PCID */ stored_hw_asid.words[0] = cap_pml4_cap_get_capPML4MappedASID_fp(newVTable); #endif #ifdef CONFIG_ARCH_AARCH64 stored_hw_asid.words[0] = cap_page_global_directory_cap_get_capPGDMappedASID(newVTable); #endif #ifdef CONFIG_ARCH_RISCV /* Get HW ASID */ stored_hw_asid.words[0] = cap_page_table_cap_get_capPTMappedASID(newVTable); #endif /* let gcc optimise this out for 1 domain */ dom = maxDom ? ksCurDomain : 0; /* ensure only the idle thread or lower prio threads are present in the scheduler */ if (likely(dest->tcbPriority < NODE_STATE(ksCurThread->tcbPriority)) && !isHighestPrio(dom, dest->tcbPriority)) { slowpath(SysCall); } /* Ensure that the endpoint has has grant or grant-reply rights so that we can * create the reply cap */ if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap) && !cap_endpoint_cap_get_capCanGrantReply(ep_cap))) { slowpath(SysCall); } #ifdef CONFIG_ARCH_AARCH32 if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysCall); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysCall); } #ifdef ENABLE_SMP_SUPPORT /* Ensure both threads have the same affinity */ if (unlikely(NODE_STATE(ksCurThread)->tcbAffinity != dest->tcbAffinity)) { slowpath(SysCall); } #endif /* ENABLE_SMP_SUPPORT */ /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.is_fastpath = true; #endif /* Dequeue the destination. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext)); if (unlikely(dest->tcbEPNext)) { dest->tcbEPNext->tcbEPPrev = NULL; } else { endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle); } badge = cap_endpoint_cap_get_capEPBadge(ep_cap); /* Block sender */ thread_state_ptr_set_tsType_np(&NODE_STATE(ksCurThread)->tcbState, ThreadState_BlockedOnReply); /* Get sender reply slot */ replySlot = TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbReply); /* Get dest caller slot */ callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller); /* Insert reply cap */ replyCanGrant = thread_state_ptr_get_blockingIPCCanGrant(&dest->tcbState);; cap_reply_cap_ptr_new_np(&callerSlot->cap, replyCanGrant, 0, TCB_REF(NODE_STATE(ksCurThread))); mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot)); mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1); fastpath_copy_mrs(length, NODE_STATE(ksCurThread), dest); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&dest->tcbState, ThreadState_Running); switchToThread_fp(dest, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); fastpath_restore(badge, msgInfo, NODE_STATE(ksCurThread)); }
exception_t invokeTCB_ThreadControl(tcb_t *target, cte_t* slot, cptr_t faultep, prio_t priority, cap_t cRoot_newCap, cte_t *cRoot_srcSlot, cap_t vRoot_newCap, cte_t *vRoot_srcSlot, word_t bufferAddr, cap_t bufferCap, cte_t *bufferSrcSlot, thread_control_flag_t updateFlags) { exception_t e; cap_t tCap = cap_thread_cap_new((word_t)target); if (updateFlags & thread_control_update_space) { target->tcbFaultHandler = faultep; } if (updateFlags & thread_control_update_priority) { setPriority(target, priority); } if (updateFlags & thread_control_update_space) { cte_t *rootSlot; rootSlot = TCB_PTR_CTE_PTR(target, tcbCTable); e = cteDelete(rootSlot, true); if (e != EXCEPTION_NONE) { return e; } if (sameObjectAs(cRoot_newCap, cRoot_srcSlot->cap) && sameObjectAs(tCap, slot->cap)) { cteInsert(cRoot_newCap, cRoot_srcSlot, rootSlot); } } if (updateFlags & thread_control_update_space) { cte_t *rootSlot; rootSlot = TCB_PTR_CTE_PTR(target, tcbVTable); e = cteDelete(rootSlot, true); if (e != EXCEPTION_NONE) { return e; } if (sameObjectAs(vRoot_newCap, vRoot_srcSlot->cap) && sameObjectAs(tCap, slot->cap)) { cteInsert(vRoot_newCap, vRoot_srcSlot, rootSlot); } } if (updateFlags & thread_control_update_ipc_buffer) { cte_t *bufferSlot; bufferSlot = TCB_PTR_CTE_PTR(target, tcbBuffer); e = cteDelete(bufferSlot, true); if (e != EXCEPTION_NONE) { return e; } target->tcbIPCBuffer = bufferAddr; if (bufferSrcSlot && sameObjectAs(bufferCap, bufferSrcSlot->cap) && sameObjectAs(tCap, slot->cap)) { cteInsert(bufferCap, bufferSrcSlot, bufferSlot); } } return EXCEPTION_NONE; }
exception_t decodeSetSpace(cap_t cap, unsigned int length, cte_t* slot, extra_caps_t extraCaps, word_t *buffer) { cptr_t faultEP; word_t cRootData, vRootData; cte_t *cRootSlot, *vRootSlot; cap_t cRootCap, vRootCap; deriveCap_ret_t dc_ret; if (length < 3 || extraCaps.excaprefs[0] == NULL || extraCaps.excaprefs[1] == NULL) { userError("TCB SetSpace: Truncated message."); current_syscall_error.type = seL4_TruncatedMessage; return EXCEPTION_SYSCALL_ERROR; } faultEP = getSyscallArg(0, buffer); cRootData = getSyscallArg(1, buffer); vRootData = getSyscallArg(2, buffer); cRootSlot = extraCaps.excaprefs[0]; cRootCap = extraCaps.excaprefs[0]->cap; vRootSlot = extraCaps.excaprefs[1]; vRootCap = extraCaps.excaprefs[1]->cap; if (slotCapLongRunningDelete( TCB_PTR_CTE_PTR(cap_thread_cap_get_capTCBPtr(cap), tcbCTable)) || slotCapLongRunningDelete( TCB_PTR_CTE_PTR(cap_thread_cap_get_capTCBPtr(cap), tcbVTable))) { userError("TCB SetSpace: CSpace or VSpace currently being deleted."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } if (cRootData != 0) { cRootCap = updateCapData(false, cRootData, cRootCap); } dc_ret = deriveCap(cRootSlot, cRootCap); if (dc_ret.status != EXCEPTION_NONE) { return dc_ret.status; } cRootCap = dc_ret.cap; if (cap_get_capType(cRootCap) != cap_cnode_cap && (!config_set(CONFIG_ALLOW_NULL_CSPACE) || cap_get_capType(cRootCap) != cap_null_cap)) { userError("TCB SetSpace: Invalid CNode cap."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } if (vRootData != 0) { vRootCap = updateCapData(false, vRootData, vRootCap); } dc_ret = deriveCap(vRootSlot, vRootCap); if (dc_ret.status != EXCEPTION_NONE) { return dc_ret.status; } vRootCap = dc_ret.cap; if (!isValidVTableRoot(vRootCap)) { userError("TCB SetSpace: Invalid VSpace cap."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } setThreadState(ksCurThread, ThreadState_Restart); return invokeTCB_ThreadControl( TCB_PTR(cap_thread_cap_get_capTCBPtr(cap)), slot, faultEP, 0, /* used to be prioInvalid, but it doesn't matter */ cRootCap, cRootSlot, vRootCap, vRootSlot, 0, cap_null_cap_new(), NULL, thread_control_update_space); }
exception_t decodeTCBConfigure(cap_t cap, unsigned int length, cte_t* slot, extra_caps_t rootCaps, word_t *buffer) { cte_t *bufferSlot, *cRootSlot, *vRootSlot; cap_t bufferCap, cRootCap, vRootCap; deriveCap_ret_t dc_ret; cptr_t faultEP; unsigned int prio; word_t cRootData, vRootData, bufferAddr; if (length < 5 || rootCaps.excaprefs[0] == NULL || rootCaps.excaprefs[1] == NULL || rootCaps.excaprefs[2] == NULL) { userError("TCB Configure: Truncated message."); current_syscall_error.type = seL4_TruncatedMessage; return EXCEPTION_SYSCALL_ERROR; } faultEP = getSyscallArg(0, buffer); prio = getSyscallArg(1, buffer); cRootData = getSyscallArg(2, buffer); vRootData = getSyscallArg(3, buffer); bufferAddr = getSyscallArg(4, buffer); cRootSlot = rootCaps.excaprefs[0]; cRootCap = rootCaps.excaprefs[0]->cap; vRootSlot = rootCaps.excaprefs[1]; vRootCap = rootCaps.excaprefs[1]->cap; bufferSlot = rootCaps.excaprefs[2]; bufferCap = rootCaps.excaprefs[2]->cap; prio = prio & MASK(8); if (prio > ksCurThread->tcbPriority) { userError("TCB Configure: Requested priority %d too high (max %d).", (int)prio, (int)(ksCurThread->tcbPriority)); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } if (bufferAddr == 0) { bufferSlot = NULL; } else { exception_t e; dc_ret = deriveCap(bufferSlot, bufferCap); if (dc_ret.status != EXCEPTION_NONE) { return dc_ret.status; } bufferCap = dc_ret.cap; e = checkValidIPCBuffer(bufferAddr, bufferCap); if (e != EXCEPTION_NONE) { return e; } } if (slotCapLongRunningDelete( TCB_PTR_CTE_PTR(cap_thread_cap_get_capTCBPtr(cap), tcbCTable)) || slotCapLongRunningDelete( TCB_PTR_CTE_PTR(cap_thread_cap_get_capTCBPtr(cap), tcbVTable))) { userError("TCB Configure: CSpace or VSpace currently being deleted."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } if (cRootData != 0) { cRootCap = updateCapData(false, cRootData, cRootCap); } dc_ret = deriveCap(cRootSlot, cRootCap); if (dc_ret.status != EXCEPTION_NONE) { return dc_ret.status; } cRootCap = dc_ret.cap; if (cap_get_capType(cRootCap) != cap_cnode_cap && (!config_set(CONFIG_ALLOW_NULL_CSPACE) || cap_get_capType(cRootCap) != cap_null_cap)) { userError("TCB Configure: CSpace cap is invalid."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } if (vRootData != 0) { vRootCap = updateCapData(false, vRootData, vRootCap); } dc_ret = deriveCap(vRootSlot, vRootCap); if (dc_ret.status != EXCEPTION_NONE) { return dc_ret.status; } vRootCap = dc_ret.cap; if (!isValidVTableRoot(vRootCap)) { userError("TCB Configure: VSpace cap is invalid."); current_syscall_error.type = seL4_IllegalOperation; return EXCEPTION_SYSCALL_ERROR; } setThreadState(ksCurThread, ThreadState_Restart); return invokeTCB_ThreadControl( TCB_PTR(cap_thread_cap_get_capTCBPtr(cap)), slot, faultEP, prio, cRootCap, cRootSlot, vRootCap, vRootSlot, bufferAddr, bufferCap, bufferSlot, thread_control_update_all); }
void #ifdef ARCH_X86 NORETURN #endif fastpath_call(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; tcb_t *dest; word_t badge; cte_t *replySlot, *callerSlot; cap_t newVTable; pde_t *cap_pd; pde_t stored_hw_asid; word_t fault_type; /* Get message info, length, and fault type. */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = fault_get_faultType(ksCurThread->tcbFault); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.path = Entry_Syscall; ksKernelEntry.syscall_no = SysCall; ksKernelEntry.cap_type = cap_endpoint_cap; ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info); ksKernelEntry.is_fastpath = true; benchmark_track_start(); #endif #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION benchmark_utilisation_kentry_stamp(); #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != fault_null_fault)) { slowpath(SysCall); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanSend(ep_cap))) { slowpath(SysCall); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Get the destination thread, which is only going to be valid * if the endpoint is valid. */ dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr)); /* Check that there's a thread waiting to receive */ if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) { slowpath(SysCall); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap; /* Get vspace root. */ #if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING) cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable)); #else cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable)); #endif /* Ensure that the destination has a valid VTable. */ if (unlikely(! isValidVTableRoot_fp(newVTable))) { slowpath(SysCall); } #ifdef ARCH_ARM /* Get HW ASID */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif /* Ensure the destination has a higher/equal priority to us. */ if (unlikely(dest->tcbPriority < ksCurThread->tcbPriority)) { slowpath(SysCall); } /* Ensure that the endpoint has has grant rights so that we can * create the reply cap */ if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap))) { slowpath(SysCall); } #ifdef ARCH_ARM if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysCall); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysCall); } /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef ARCH_X86 /* Need to update NextIP in the calling thread */ setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2); #endif /* Dequeue the destination. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext)); if (unlikely(dest->tcbEPNext)) { dest->tcbEPNext->tcbEPPrev = NULL; } else { endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle); } badge = cap_endpoint_cap_get_capEPBadge(ep_cap); /* Block sender */ thread_state_ptr_set_tsType_np(&ksCurThread->tcbState, ThreadState_BlockedOnReply); /* Get sender reply slot */ replySlot = TCB_PTR_CTE_PTR(ksCurThread, tcbReply); /* Get dest caller slot */ callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller); /* Insert reply cap */ cap_reply_cap_ptr_new_np(&callerSlot->cap, 0, TCB_REF(ksCurThread)); mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot)); mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1); fastpath_copy_mrs (length, ksCurThread, dest); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&dest->tcbState, ThreadState_Running); switchToThread_fp(dest, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES benchmark_track_exit(); #endif fastpath_restore(badge, msgInfo, ksCurThread); }
void fastpath_reply_recv(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; cte_t *callerSlot; cap_t callerCap; tcb_t *caller; word_t badge; tcb_t *endpointTail; word_t fault_type; cap_t newVTable; pde_t *cap_pd; pde_t stored_hw_asid; /* Get message info and length */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = fault_get_faultType(ksCurThread->tcbFault); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.path = Entry_Syscall; ksKernelEntry.syscall_no = SysReplyRecv; ksKernelEntry.cap_type = cap_endpoint_cap; ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info); ksKernelEntry.is_fastpath = true; benchmark_track_start(); #endif #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION benchmark_utilisation_kentry_stamp(); #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != fault_null_fault)) { slowpath(SysReplyRecv); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanReceive(ep_cap))) { slowpath(SysReplyRecv); } /* Check there is nothing waiting on the notification */ if (ksCurThread->tcbBoundNotification && notification_ptr_get_state(ksCurThread->tcbBoundNotification) == NtfnState_Active) { slowpath(SysReplyRecv); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Check that there's not a thread waiting to send */ if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) { slowpath(SysReplyRecv); } /* Only reply if the reply cap is valid. */ callerSlot = TCB_PTR_CTE_PTR(ksCurThread, tcbCaller); callerCap = callerSlot->cap; if (unlikely(!fastpath_reply_cap_check(callerCap))) { slowpath(SysReplyRecv); } /* Determine who the caller is. */ caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* Check that the caller has not faulted, in which case a fault reply is generated instead. */ fault_type = fault_get_faultType(caller->tcbFault); if (unlikely(fault_type != fault_null_fault)) { slowpath(SysReplyRecv); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap; /* Get vspace root. */ #if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING) cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable)); #else cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable)); #endif /* Ensure that the destination has a valid MMU. */ if (unlikely(! isValidVTableRoot_fp (newVTable))) { slowpath(SysReplyRecv); } #ifdef ARCH_ARM /* Get HWASID. */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif /* Ensure the original caller can be scheduled directly. */ if (unlikely(caller->tcbPriority < ksCurThread->tcbPriority)) { slowpath(SysReplyRecv); } #ifdef ARCH_ARM /* Ensure the HWASID is valid. */ if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysReplyRecv); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysReplyRecv); } /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef ARCH_X86 /* Need to update NextIP in the calling thread */ setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2); #endif /* Set thread state to BlockedOnReceive */ thread_state_ptr_mset_blockingObject_tsType( &ksCurThread->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive); /* Place the thread in the endpoint queue */ endpointTail = TCB_PTR(endpoint_ptr_get_epQueue_tail(ep_ptr)); if (likely(!endpointTail)) { ksCurThread->tcbEPPrev = NULL; ksCurThread->tcbEPNext = NULL; /* Set head/tail of queue and endpoint state. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(ksCurThread)); endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread), EPState_Recv); } else { /* Append current thread onto the queue. */ endpointTail->tcbEPNext = ksCurThread; ksCurThread->tcbEPPrev = endpointTail; ksCurThread->tcbEPNext = NULL; /* Update tail of queue. */ endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread), EPState_Recv); } /* Delete the reply cap. */ mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode, 0, 1, 1); callerSlot->cap = cap_null_cap_new(); callerSlot->cteMDBNode = nullMDBNode; /* I know there's no fault, so straight to the transfer. */ /* Replies don't have a badge. */ badge = 0; fastpath_copy_mrs (length, ksCurThread, caller); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&caller->tcbState, ThreadState_Running); switchToThread_fp(caller, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES benchmark_track_exit(); #endif fastpath_restore(badge, msgInfo, ksCurThread); }