void sendSignal(notification_t *ntfnPtr, word_t badge) { switch (notification_ptr_get_state(ntfnPtr)) { case NtfnState_Idle: { tcb_t *tcb = (tcb_t*)notification_ptr_get_ntfnBoundTCB(ntfnPtr); /* Check if we are bound and that thread is waiting for a message */ if (tcb) { if (thread_state_ptr_get_tsType(&tcb->tcbState) == ThreadState_BlockedOnReceive) { /* Send and start thread running */ cancelIPC(tcb); setThreadState(tcb, ThreadState_Running); setRegister(tcb, badgeRegister, badge); switchIfRequiredTo(tcb); } else { ntfn_set_active(ntfnPtr, badge); } } else { ntfn_set_active(ntfnPtr, badge); } break; } case NtfnState_Waiting: { tcb_queue_t ntfn_queue; tcb_t *dest; ntfn_queue = ntfn_ptr_get_queue(ntfnPtr); dest = ntfn_queue.head; /* Haskell error "WaitingNtfn Notification must have non-empty queue" */ assert(dest); /* Dequeue TCB */ ntfn_queue = tcbEPDequeue(dest, ntfn_queue); ntfn_ptr_set_queue(ntfnPtr, ntfn_queue); /* set the thread state to idle if the queue is empty */ if (!ntfn_queue.head) { notification_ptr_set_state(ntfnPtr, NtfnState_Idle); } setThreadState(dest, ThreadState_Running); setRegister(dest, badgeRegister, badge); switchIfRequiredTo(dest); break; } case NtfnState_Active: { word_t badge2; badge2 = notification_ptr_get_ntfnMsgIdentifier(ntfnPtr); badge2 |= badge; notification_ptr_set_ntfnMsgIdentifier(ntfnPtr, badge2); break; } } }
void completeSignal(notification_t *ntfnPtr, tcb_t *tcb) { word_t badge; if (likely(tcb && notification_ptr_get_state(ntfnPtr) == NtfnState_Active)) { badge = notification_ptr_get_ntfnMsgIdentifier(ntfnPtr); setRegister(tcb, badgeRegister, badge); notification_ptr_set_state(ntfnPtr, NtfnState_Idle); } else { fail("tried to complete signal with inactive notification object"); } }
void cancelAllSignals(notification_t *ntfnPtr) { if (notification_ptr_get_state(ntfnPtr) == NtfnState_Waiting) { tcb_t *thread = TCB_PTR(notification_ptr_get_ntfnQueue_head(ntfnPtr)); notification_ptr_set_state(ntfnPtr, NtfnState_Idle); notification_ptr_set_ntfnQueue_head(ntfnPtr, 0); notification_ptr_set_ntfnQueue_tail(ntfnPtr, 0); /* Set all waiting threads to Restart */ for (; thread; thread = thread->tcbEPNext) { setThreadState(thread, ThreadState_Restart); SCHED_ENQUEUE(thread); } rescheduleRequired(); } }
void cancelSignal(tcb_t *threadPtr, notification_t *ntfnPtr) { tcb_queue_t ntfn_queue; /* Haskell error "cancelSignal: notification object must be in a waiting" state */ assert(notification_ptr_get_state(ntfnPtr) == NtfnState_Waiting); /* Dequeue TCB */ ntfn_queue = ntfn_ptr_get_queue(ntfnPtr); ntfn_queue = tcbEPDequeue(threadPtr, ntfn_queue); ntfn_ptr_set_queue(ntfnPtr, ntfn_queue); /* Make notification object idle */ if (!ntfn_queue.head) { notification_ptr_set_state(ntfnPtr, NtfnState_Idle); } /* Make thread inactive */ setThreadState(threadPtr, ThreadState_Inactive); }
void receiveSignal(tcb_t *thread, cap_t cap, bool_t isBlocking) { notification_t *ntfnPtr; ntfnPtr = NTFN_PTR(cap_notification_cap_get_capNtfnPtr(cap)); switch (notification_ptr_get_state(ntfnPtr)) { case NtfnState_Idle: case NtfnState_Waiting: { tcb_queue_t ntfn_queue; if (isBlocking) { /* Block thread on notification object */ thread_state_ptr_set_tsType(&thread->tcbState, ThreadState_BlockedOnNotification); thread_state_ptr_set_blockingObject(&thread->tcbState, NTFN_REF(ntfnPtr)); scheduleTCB(thread); /* Enqueue TCB */ ntfn_queue = ntfn_ptr_get_queue(ntfnPtr); ntfn_queue = tcbEPAppend(thread, ntfn_queue); notification_ptr_set_state(ntfnPtr, NtfnState_Waiting); ntfn_ptr_set_queue(ntfnPtr, ntfn_queue); } else { doNBRecvFailedTransfer(thread); } break; } case NtfnState_Active: setRegister( thread, badgeRegister, notification_ptr_get_ntfnMsgIdentifier(ntfnPtr)); notification_ptr_set_state(ntfnPtr, NtfnState_Idle); break; } }
void fastpath_reply_recv(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; cte_t *callerSlot; cap_t callerCap; tcb_t *caller; word_t badge; tcb_t *endpointTail; word_t fault_type; cap_t newVTable; vspace_root_t *cap_pd; pde_t stored_hw_asid; dom_t dom; /* Get message info and length */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = seL4_Fault_get_seL4_FaultType(NODE_STATE(ksCurThread)->tcbFault); /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != seL4_Fault_NullFault)) { slowpath(SysReplyRecv); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanReceive(ep_cap))) { slowpath(SysReplyRecv); } /* Check there is nothing waiting on the notification */ if (NODE_STATE(ksCurThread)->tcbBoundNotification && notification_ptr_get_state(NODE_STATE(ksCurThread)->tcbBoundNotification) == NtfnState_Active) { slowpath(SysReplyRecv); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Check that there's not a thread waiting to send */ if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) { slowpath(SysReplyRecv); } /* Only reply if the reply cap is valid. */ callerSlot = TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCaller); callerCap = callerSlot->cap; if (unlikely(!fastpath_reply_cap_check(callerCap))) { slowpath(SysReplyRecv); } /* Determine who the caller is. */ caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* ensure we are not single stepping the caller in ia32 */ #if defined(CONFIG_HARDWARE_DEBUG_API) && defined(CONFIG_ARCH_IA32) if (caller->tcbArch.tcbContext.breakpointState.single_step_enabled) { slowpath(SysReplyRecv); } #endif /* Check that the caller has not faulted, in which case a fault reply is generated instead. */ fault_type = seL4_Fault_get_seL4_FaultType(caller->tcbFault); if (unlikely(fault_type != seL4_Fault_NullFault)) { slowpath(SysReplyRecv); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap; /* Get vspace root. */ cap_pd = cap_vtable_cap_get_vspace_root_fp(newVTable); /* Ensure that the destination has a valid MMU. */ if (unlikely(! isValidVTableRoot_fp(newVTable))) { slowpath(SysReplyRecv); } #ifdef CONFIG_ARCH_AARCH32 /* Get HWASID. */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif #ifdef CONFIG_ARCH_X86_64 stored_hw_asid.words[0] = cap_pml4_cap_get_capPML4MappedASID(newVTable); #endif #ifdef CONFIG_ARCH_AARCH64 stored_hw_asid.words[0] = cap_page_global_directory_cap_get_capPGDMappedASID(newVTable); #endif #ifdef CONFIG_ARCH_RISCV stored_hw_asid.words[0] = cap_page_table_cap_get_capPTMappedASID(newVTable); #endif /* Ensure the original caller can be scheduled directly. */ dom = maxDom ? ksCurDomain : 0; if (unlikely(!isHighestPrio(dom, caller->tcbPriority))) { slowpath(SysReplyRecv); } #ifdef CONFIG_ARCH_AARCH32 /* Ensure the HWASID is valid. */ if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysReplyRecv); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysReplyRecv); } #ifdef ENABLE_SMP_SUPPORT /* Ensure both threads have the same affinity */ if (unlikely(NODE_STATE(ksCurThread)->tcbAffinity != caller->tcbAffinity)) { slowpath(SysReplyRecv); } #endif /* ENABLE_SMP_SUPPORT */ /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.is_fastpath = true; #endif /* Set thread state to BlockedOnReceive */ thread_state_ptr_mset_blockingObject_tsType( &NODE_STATE(ksCurThread)->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive); thread_state_ptr_set_blockingIPCCanGrant(&NODE_STATE(ksCurThread)->tcbState, cap_endpoint_cap_get_capCanGrant(ep_cap));; /* Place the thread in the endpoint queue */ endpointTail = endpoint_ptr_get_epQueue_tail_fp(ep_ptr); if (likely(!endpointTail)) { NODE_STATE(ksCurThread)->tcbEPPrev = NULL; NODE_STATE(ksCurThread)->tcbEPNext = NULL; /* Set head/tail of queue and endpoint state. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(NODE_STATE(ksCurThread))); endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)), EPState_Recv); } else { /* Append current thread onto the queue. */ endpointTail->tcbEPNext = NODE_STATE(ksCurThread); NODE_STATE(ksCurThread)->tcbEPPrev = endpointTail; NODE_STATE(ksCurThread)->tcbEPNext = NULL; /* Update tail of queue. */ endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)), EPState_Recv); } /* Delete the reply cap. */ mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode, 0, 1, 1); callerSlot->cap = cap_null_cap_new(); callerSlot->cteMDBNode = nullMDBNode; /* I know there's no fault, so straight to the transfer. */ /* Replies don't have a badge. */ badge = 0; fastpath_copy_mrs(length, NODE_STATE(ksCurThread), caller); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&caller->tcbState, ThreadState_Running); switchToThread_fp(caller, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); fastpath_restore(badge, msgInfo, NODE_STATE(ksCurThread)); }
void receiveIPC(tcb_t *thread, cap_t cap, bool_t isBlocking) { endpoint_t *epptr; notification_t *ntfnPtr; /* Haskell error "receiveIPC: invalid cap" */ assert(cap_get_capType(cap) == cap_endpoint_cap); epptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(cap)); /* Check for anything waiting in the notification */ ntfnPtr = thread->tcbBoundNotification; if (ntfnPtr && notification_ptr_get_state(ntfnPtr) == NtfnState_Active) { completeSignal(ntfnPtr, thread); } else { switch (endpoint_ptr_get_state(epptr)) { case EPState_Idle: case EPState_Recv: { tcb_queue_t queue; if (isBlocking) { /* Set thread state to BlockedOnReceive */ thread_state_ptr_set_tsType(&thread->tcbState, ThreadState_BlockedOnReceive); thread_state_ptr_set_blockingObject( &thread->tcbState, EP_REF(epptr)); scheduleTCB(thread); /* Place calling thread in endpoint queue */ queue = ep_ptr_get_queue(epptr); queue = tcbEPAppend(thread, queue); endpoint_ptr_set_state(epptr, EPState_Recv); ep_ptr_set_queue(epptr, queue); } else { doNBRecvFailedTransfer(thread); } break; } case EPState_Send: { tcb_queue_t queue; tcb_t *sender; word_t badge; bool_t canGrant; bool_t do_call; /* Get the head of the endpoint queue. */ queue = ep_ptr_get_queue(epptr); sender = queue.head; /* Haskell error "Send endpoint queue must not be empty" */ assert(sender); /* Dequeue the first TCB */ queue = tcbEPDequeue(sender, queue); ep_ptr_set_queue(epptr, queue); if (!queue.head) { endpoint_ptr_set_state(epptr, EPState_Idle); } /* Get sender IPC details */ badge = thread_state_ptr_get_blockingIPCBadge(&sender->tcbState); canGrant = thread_state_ptr_get_blockingIPCCanGrant(&sender->tcbState); /* Do the transfer */ doIPCTransfer(sender, epptr, badge, canGrant, thread); do_call = thread_state_ptr_get_blockingIPCIsCall(&sender->tcbState); if (do_call || seL4_Fault_get_seL4_FaultType(sender->tcbFault) != seL4_Fault_NullFault) { if (canGrant) { setupCallerCap(sender, thread); } else { setThreadState(sender, ThreadState_Inactive); } } else { setThreadState(sender, ThreadState_Running); switchIfRequiredTo(sender); } break; } } } }
void sendSignal(notification_t *ntfnPtr, word_t badge) { switch (notification_ptr_get_state(ntfnPtr)) { case NtfnState_Idle: { tcb_t *tcb = (tcb_t *)notification_ptr_get_ntfnBoundTCB(ntfnPtr); /* Check if we are bound and that thread is waiting for a message */ if (tcb) { if (thread_state_ptr_get_tsType(&tcb->tcbState) == ThreadState_BlockedOnReceive) { /* Send and start thread running */ cancelIPC(tcb); setThreadState(tcb, ThreadState_Running); setRegister(tcb, badgeRegister, badge); possibleSwitchTo(tcb); #ifdef CONFIG_VTX } else if (thread_state_ptr_get_tsType(&tcb->tcbState) == ThreadState_RunningVM) { #ifdef ENABLE_SMP_SUPPORT if (tcb->tcbAffinity != getCurrentCPUIndex()) { ntfn_set_active(ntfnPtr, badge); doRemoteVMCheckBoundNotification(tcb->tcbAffinity, tcb); } else #endif /* ENABLE_SMP_SUPPORT */ { setThreadState(tcb, ThreadState_Running); setRegister(tcb, badgeRegister, badge); Arch_leaveVMAsyncTransfer(tcb); possibleSwitchTo(tcb); } #endif /* CONFIG_VTX */ } else { /* In particular, this path is taken when a thread * is waiting on a reply cap since BlockedOnReply * would also trigger this path. I.e, a thread * with a bound notification will not be awakened * by signals on that bound notification if it is * in the middle of an seL4_Call. */ ntfn_set_active(ntfnPtr, badge); } } else { ntfn_set_active(ntfnPtr, badge); } break; } case NtfnState_Waiting: { tcb_queue_t ntfn_queue; tcb_t *dest; ntfn_queue = ntfn_ptr_get_queue(ntfnPtr); dest = ntfn_queue.head; /* Haskell error "WaitingNtfn Notification must have non-empty queue" */ assert(dest); /* Dequeue TCB */ ntfn_queue = tcbEPDequeue(dest, ntfn_queue); ntfn_ptr_set_queue(ntfnPtr, ntfn_queue); /* set the thread state to idle if the queue is empty */ if (!ntfn_queue.head) { notification_ptr_set_state(ntfnPtr, NtfnState_Idle); } setThreadState(dest, ThreadState_Running); setRegister(dest, badgeRegister, badge); possibleSwitchTo(dest); break; } case NtfnState_Active: { word_t badge2; badge2 = notification_ptr_get_ntfnMsgIdentifier(ntfnPtr); badge2 |= badge; notification_ptr_set_ntfnMsgIdentifier(ntfnPtr, badge2); break; } } }
void fastpath_reply_recv(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; cte_t *callerSlot; cap_t callerCap; tcb_t *caller; word_t badge; tcb_t *endpointTail; word_t fault_type; cap_t newVTable; pde_t *cap_pd; pde_t stored_hw_asid; /* Get message info and length */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = fault_get_faultType(ksCurThread->tcbFault); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.path = Entry_Syscall; ksKernelEntry.syscall_no = SysReplyRecv; ksKernelEntry.cap_type = cap_endpoint_cap; ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info); ksKernelEntry.is_fastpath = true; benchmark_track_start(); #endif #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION benchmark_utilisation_kentry_stamp(); #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != fault_null_fault)) { slowpath(SysReplyRecv); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanReceive(ep_cap))) { slowpath(SysReplyRecv); } /* Check there is nothing waiting on the notification */ if (ksCurThread->tcbBoundNotification && notification_ptr_get_state(ksCurThread->tcbBoundNotification) == NtfnState_Active) { slowpath(SysReplyRecv); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Check that there's not a thread waiting to send */ if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) { slowpath(SysReplyRecv); } /* Only reply if the reply cap is valid. */ callerSlot = TCB_PTR_CTE_PTR(ksCurThread, tcbCaller); callerCap = callerSlot->cap; if (unlikely(!fastpath_reply_cap_check(callerCap))) { slowpath(SysReplyRecv); } /* Determine who the caller is. */ caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* Check that the caller has not faulted, in which case a fault reply is generated instead. */ fault_type = fault_get_faultType(caller->tcbFault); if (unlikely(fault_type != fault_null_fault)) { slowpath(SysReplyRecv); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap; /* Get vspace root. */ #if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING) cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable)); #else cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable)); #endif /* Ensure that the destination has a valid MMU. */ if (unlikely(! isValidVTableRoot_fp (newVTable))) { slowpath(SysReplyRecv); } #ifdef ARCH_ARM /* Get HWASID. */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif /* Ensure the original caller can be scheduled directly. */ if (unlikely(caller->tcbPriority < ksCurThread->tcbPriority)) { slowpath(SysReplyRecv); } #ifdef ARCH_ARM /* Ensure the HWASID is valid. */ if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysReplyRecv); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysReplyRecv); } /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef ARCH_X86 /* Need to update NextIP in the calling thread */ setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2); #endif /* Set thread state to BlockedOnReceive */ thread_state_ptr_mset_blockingObject_tsType( &ksCurThread->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive); /* Place the thread in the endpoint queue */ endpointTail = TCB_PTR(endpoint_ptr_get_epQueue_tail(ep_ptr)); if (likely(!endpointTail)) { ksCurThread->tcbEPPrev = NULL; ksCurThread->tcbEPNext = NULL; /* Set head/tail of queue and endpoint state. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(ksCurThread)); endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread), EPState_Recv); } else { /* Append current thread onto the queue. */ endpointTail->tcbEPNext = ksCurThread; ksCurThread->tcbEPPrev = endpointTail; ksCurThread->tcbEPNext = NULL; /* Update tail of queue. */ endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread), EPState_Recv); } /* Delete the reply cap. */ mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode, 0, 1, 1); callerSlot->cap = cap_null_cap_new(); callerSlot->cteMDBNode = nullMDBNode; /* I know there's no fault, so straight to the transfer. */ /* Replies don't have a badge. */ badge = 0; fastpath_copy_mrs (length, ksCurThread, caller); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&caller->tcbState, ThreadState_Running); switchToThread_fp(caller, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES benchmark_track_exit(); #endif fastpath_restore(badge, msgInfo, ksCurThread); }