void doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot) { assert(thread_state_get_tsType(receiver->tcbState) == ThreadState_BlockedOnReply); if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) { doIPCTransfer(sender, NULL, 0, true, receiver, false); setThreadState(receiver, ThreadState_Running); attemptSwitchTo(receiver); } else { bool_t restart; restart = handleFaultReply(receiver, sender); fault_null_fault_ptr_new(&receiver->tcbFault); if (restart) { setThreadState(receiver, ThreadState_Restart); attemptSwitchTo(receiver); } else { setThreadState(receiver, ThreadState_Inactive); } } if (cap_reply_cap_get_capInCDT(slot->cap)) { cte_t *replySlot = TCB_PTR_CTE_PTR(receiver, tcbReply); assert(cap_get_capType(replySlot->cap) == cap_reply_cap); assert(cap_reply_cap_get_capInCDT(replySlot->cap)); cdtRemove(replySlot); cdtRemove(slot); slot->cap = cap_null_cap_new(); replySlot->cap = cap_reply_cap_new(false, true, TCB_REF(NULL)); } else { deleteCallerCap(sender); } }
void doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot) { assert(thread_state_get_tsType(receiver->tcbState) == ThreadState_BlockedOnReply); if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) { doIPCTransfer(sender, NULL, 0, true, receiver, false); /** GHOSTUPD: "(True, gs_set_assn cteDeleteOne_'proc (ucast cap_reply_cap))" */ cteDeleteOne(slot); setThreadState(receiver, ThreadState_Running); attemptSwitchTo(receiver); } else { bool_t restart; /** GHOSTUPD: "(True, gs_set_assn cteDeleteOne_'proc (ucast cap_reply_cap))" */ cteDeleteOne(slot); restart = handleFaultReply(receiver, sender); fault_null_fault_ptr_new(&receiver->tcbFault); if (restart) { setThreadState(receiver, ThreadState_Restart); attemptSwitchTo(receiver); } else { setThreadState(receiver, ThreadState_Inactive); } } }
void doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot) { assert(thread_state_get_tsType(receiver->tcbState) == ThreadState_BlockedOnReply); if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) { doIPCTransfer(sender, NULL, 0, true, receiver, false); setThreadState(receiver, ThreadState_Running); attemptSwitchTo(receiver); } else { bool_t restart; restart = handleFaultReply(receiver, sender); fault_null_fault_ptr_new(&receiver->tcbFault); if (restart) { setThreadState(receiver, ThreadState_Restart); attemptSwitchTo(receiver); } else { setThreadState(receiver, ThreadState_Inactive); } } finaliseCap(slot->cap, true, true); slot->cap = cap_null_cap_new(); }
void doFaultTransfer(word_t badge, tcb_t *sender, tcb_t *receiver, word_t *receiverIPCBuffer) { unsigned int sent; message_info_t msgInfo; sent = setMRs_fault(sender, receiver, receiverIPCBuffer); msgInfo = message_info_new( fault_get_faultType(sender->tcbFault), 0, 0, sent); setRegister(receiver, msgInfoRegister, wordFromMessageInfo(msgInfo)); setRegister(receiver, badgeRegister, badge); }
void doIPCTransfer(tcb_t *sender, endpoint_t *endpoint, word_t badge, bool_t grant, tcb_t *receiver, bool_t diminish) { void *receiveBuffer, *sendBuffer; receiveBuffer = lookupIPCBuffer(true, receiver); if (likely(!fault_get_faultType(sender->tcbFault) != fault_null_fault)) { sendBuffer = lookupIPCBuffer(false, sender); doNormalTransfer(sender, sendBuffer, endpoint, badge, grant, receiver, receiveBuffer, diminish); } else { doFaultTransfer(badge, sender, receiver, receiveBuffer); } }
void receiveIPC(tcb_t *thread, cap_t cap) { endpoint_t *epptr; bool_t diminish; async_endpoint_t *aepptr; /* Haskell error "receiveIPC: invalid cap" */ assert(cap_get_capType(cap) == cap_endpoint_cap); //printf("\n;;;;;;;;;In function receiveIPC;;;;;;;\n"); epptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(cap)); diminish = !cap_endpoint_cap_get_capCanSend(cap); /* Check for anything waiting in the async endpoint*/ aepptr = thread->boundAsyncEndpoint; if (aepptr && async_endpoint_ptr_get_state(aepptr) == AEPState_Active) { completeAsyncIPC(aepptr, thread); } else { switch (endpoint_ptr_get_state(epptr)) { case EPState_Idle: //printf("in case idle\n"); case EPState_Recv: { tcb_queue_t queue; //printf("in case recv\n"); /* Set thread state to BlockedOnReceive */ thread_state_ptr_set_tsType(&thread->tcbState, ThreadState_BlockedOnReceive); thread_state_ptr_set_blockingIPCEndpoint( &thread->tcbState, EP_REF(epptr)); thread_state_ptr_set_blockingIPCDiminishCaps( &thread->tcbState, diminish); scheduleTCB(thread); /* Place calling thread in endpoint queue */ queue = ep_ptr_get_queue(epptr); queue = tcbEPAppend(thread, queue); endpoint_ptr_set_state(epptr, EPState_Recv); ep_ptr_set_queue(epptr, queue); break; } case EPState_Send: { tcb_queue_t queue; tcb_t *sender; word_t badge; bool_t canGrant; bool_t do_call; //printf("in case send\n"); /* Get the head of the endpoint queue. */ queue = ep_ptr_get_queue(epptr); sender = queue.head; /* Haskell error "Send endpoint queue must not be empty" */ assert(sender); /* Dequeue the first TCB */ queue = tcbEPDequeue(sender, queue); ep_ptr_set_queue(epptr, queue); if (!queue.head) { endpoint_ptr_set_state(epptr, EPState_Idle); } /* Get sender IPC details */ badge = thread_state_ptr_get_blockingIPCBadge(&sender->tcbState); canGrant = thread_state_ptr_get_blockingIPCCanGrant(&sender->tcbState); /* Do the transfer */ doIPCTransfer(sender, epptr, badge, canGrant, thread, diminish); do_call = thread_state_ptr_get_blockingIPCIsCall(&sender->tcbState); if (do_call || fault_get_faultType(sender->tcbFault) != fault_null_fault) { if (canGrant && !diminish) { setupCallerCap(sender, thread); } else { setThreadState(sender, ThreadState_Inactive); } } else { setThreadState(sender, ThreadState_Running); switchIfRequiredTo(sender); } break; } } } }
bool_t handleFaultReply(tcb_t *receiver, tcb_t *sender) { message_info_t tag; word_t label; fault_t fault; unsigned int length; /* These lookups are moved inward from doReplyTransfer */ tag = messageInfoFromWord(getRegister(sender, msgInfoRegister)); label = message_info_get_msgLabel(tag); length = message_info_get_msgLength(tag); fault = receiver->tcbFault; switch (fault_get_faultType(fault)) { case fault_cap_fault: return true; case fault_vm_fault: return true; case fault_unknown_syscall: { unsigned int i; register_t r; word_t v; word_t *sendBuf; sendBuf = lookupIPCBuffer(false, sender); /* Assumes n_syscallMessage > n_msgRegisters */ for (i = 0; i < length && i < n_msgRegisters; i++) { r = syscallMessage[i]; v = getRegister(sender, msgRegisters[i]); setRegister(receiver, r, sanitiseRegister(r, v)); } if (sendBuf) { for (; i < length && i < n_syscallMessage; i++) { r = syscallMessage[i]; v = sendBuf[i + 1]; setRegister(receiver, r, sanitiseRegister(r, v)); } } } return (label == 0); case fault_user_exception: { unsigned int i; register_t r; word_t v; /* Assumes n_exceptionMessage <= n_msgRegisters */ for (i = 0; i < length && i < n_exceptionMessage; i++) { r = exceptionMessage[i]; v = getRegister(sender, msgRegisters[i]); setRegister(receiver, r, sanitiseRegister(r, v)); } } return (label == 0); default: fail("Invalid fault"); } }
unsigned int setMRs_fault(tcb_t *sender, tcb_t* receiver, word_t *receiveIPCBuffer) { assert(n_msgRegisters == 2); switch (fault_get_faultType(sender->tcbFault)) { case fault_cap_fault: setRegister(receiver, msgRegisters[0], getRestartPC(sender)); setRegister(receiver, msgRegisters[1], fault_cap_fault_get_address(sender->tcbFault)); if (!receiveIPCBuffer) { return n_msgRegisters; } receiveIPCBuffer[2 + 1] = fault_cap_fault_get_inReceivePhase(sender->tcbFault); return setMRs_lookup_failure(receiver, receiveIPCBuffer, sender->tcbLookupFailure, 3); case fault_vm_fault: setRegister(receiver, msgRegisters[0], getRestartPC(sender)); setRegister(receiver, msgRegisters[1], fault_vm_fault_get_address(sender->tcbFault)); if (!receiveIPCBuffer) { return n_msgRegisters; } receiveIPCBuffer[2 + 1] = fault_vm_fault_get_instructionFault(sender->tcbFault); receiveIPCBuffer[3 + 1] = fault_vm_fault_get_FSR(sender->tcbFault); return 4; case fault_unknown_syscall: { unsigned int i; for (i = 0; i < n_msgRegisters; i++) { setRegister(receiver, msgRegisters[i], getRegister(sender, syscallMessage[i])); } if (receiveIPCBuffer) { for (; i < n_syscallMessage; i++) { receiveIPCBuffer[i + 1] = getRegister(sender, syscallMessage[i]); } receiveIPCBuffer[i + 1] = fault_unknown_syscall_get_syscallNumber(sender->tcbFault); return n_syscallMessage + 1; } else { return n_msgRegisters; } } case fault_user_exception: { unsigned int i; for (i = 0; i < n_msgRegisters; i++) { setRegister(receiver, msgRegisters[i], getRegister(sender, exceptionMessage[i])); } if (receiveIPCBuffer) { for (; i < n_exceptionMessage; i++) { receiveIPCBuffer[i + 1] = getRegister(sender, exceptionMessage[i]); } receiveIPCBuffer[n_exceptionMessage + 1] = fault_user_exception_get_number(sender->tcbFault); receiveIPCBuffer[n_exceptionMessage + 2] = fault_user_exception_get_code(sender->tcbFault); return n_exceptionMessage + 2; } else { return n_msgRegisters; } } default: fail("Invalid fault"); } }
void fastpath_reply_recv(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; cte_t *callerSlot; cap_t callerCap; tcb_t *caller; word_t badge; tcb_t *endpointTail; word_t fault_type; cap_t newVTable; pde_t *cap_pd; pde_t stored_hw_asid; /* Get message info and length */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = fault_get_faultType(ksCurThread->tcbFault); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.path = Entry_Syscall; ksKernelEntry.syscall_no = SysReplyRecv; ksKernelEntry.cap_type = cap_endpoint_cap; ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info); ksKernelEntry.is_fastpath = true; benchmark_track_start(); #endif #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION benchmark_utilisation_kentry_stamp(); #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != fault_null_fault)) { slowpath(SysReplyRecv); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanReceive(ep_cap))) { slowpath(SysReplyRecv); } /* Check there is nothing waiting on the notification */ if (ksCurThread->tcbBoundNotification && notification_ptr_get_state(ksCurThread->tcbBoundNotification) == NtfnState_Active) { slowpath(SysReplyRecv); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Check that there's not a thread waiting to send */ if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) { slowpath(SysReplyRecv); } /* Only reply if the reply cap is valid. */ callerSlot = TCB_PTR_CTE_PTR(ksCurThread, tcbCaller); callerCap = callerSlot->cap; if (unlikely(!fastpath_reply_cap_check(callerCap))) { slowpath(SysReplyRecv); } /* Determine who the caller is. */ caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap)); /* Check that the caller has not faulted, in which case a fault reply is generated instead. */ fault_type = fault_get_faultType(caller->tcbFault); if (unlikely(fault_type != fault_null_fault)) { slowpath(SysReplyRecv); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap; /* Get vspace root. */ #if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING) cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable)); #else cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable)); #endif /* Ensure that the destination has a valid MMU. */ if (unlikely(! isValidVTableRoot_fp (newVTable))) { slowpath(SysReplyRecv); } #ifdef ARCH_ARM /* Get HWASID. */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif /* Ensure the original caller can be scheduled directly. */ if (unlikely(caller->tcbPriority < ksCurThread->tcbPriority)) { slowpath(SysReplyRecv); } #ifdef ARCH_ARM /* Ensure the HWASID is valid. */ if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysReplyRecv); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysReplyRecv); } /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef ARCH_X86 /* Need to update NextIP in the calling thread */ setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2); #endif /* Set thread state to BlockedOnReceive */ thread_state_ptr_mset_blockingObject_tsType( &ksCurThread->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive); /* Place the thread in the endpoint queue */ endpointTail = TCB_PTR(endpoint_ptr_get_epQueue_tail(ep_ptr)); if (likely(!endpointTail)) { ksCurThread->tcbEPPrev = NULL; ksCurThread->tcbEPNext = NULL; /* Set head/tail of queue and endpoint state. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(ksCurThread)); endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread), EPState_Recv); } else { /* Append current thread onto the queue. */ endpointTail->tcbEPNext = ksCurThread; ksCurThread->tcbEPPrev = endpointTail; ksCurThread->tcbEPNext = NULL; /* Update tail of queue. */ endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread), EPState_Recv); } /* Delete the reply cap. */ mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode, 0, 1, 1); callerSlot->cap = cap_null_cap_new(); callerSlot->cteMDBNode = nullMDBNode; /* I know there's no fault, so straight to the transfer. */ /* Replies don't have a badge. */ badge = 0; fastpath_copy_mrs (length, ksCurThread, caller); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&caller->tcbState, ThreadState_Running); switchToThread_fp(caller, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES benchmark_track_exit(); #endif fastpath_restore(badge, msgInfo, ksCurThread); }
void #ifdef ARCH_X86 NORETURN #endif fastpath_call(word_t cptr, word_t msgInfo) { seL4_MessageInfo_t info; cap_t ep_cap; endpoint_t *ep_ptr; word_t length; tcb_t *dest; word_t badge; cte_t *replySlot, *callerSlot; cap_t newVTable; pde_t *cap_pd; pde_t stored_hw_asid; word_t fault_type; /* Get message info, length, and fault type. */ info = messageInfoFromWord_raw(msgInfo); length = seL4_MessageInfo_get_length(info); fault_type = fault_get_faultType(ksCurThread->tcbFault); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES ksKernelEntry.path = Entry_Syscall; ksKernelEntry.syscall_no = SysCall; ksKernelEntry.cap_type = cap_endpoint_cap; ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info); ksKernelEntry.is_fastpath = true; benchmark_track_start(); #endif #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION benchmark_utilisation_kentry_stamp(); #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ /* Check there's no extra caps, the length is ok and there's no * saved fault. */ if (unlikely(fastpath_mi_check(msgInfo) || fault_type != fault_null_fault)) { slowpath(SysCall); } /* Lookup the cap */ ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr); /* Check it's an endpoint */ if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) || !cap_endpoint_cap_get_capCanSend(ep_cap))) { slowpath(SysCall); } /* Get the endpoint address */ ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap)); /* Get the destination thread, which is only going to be valid * if the endpoint is valid. */ dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr)); /* Check that there's a thread waiting to receive */ if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) { slowpath(SysCall); } /* Get destination thread.*/ newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap; /* Get vspace root. */ #if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING) cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable)); #else cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable)); #endif /* Ensure that the destination has a valid VTable. */ if (unlikely(! isValidVTableRoot_fp(newVTable))) { slowpath(SysCall); } #ifdef ARCH_ARM /* Get HW ASID */ stored_hw_asid = cap_pd[PD_ASID_SLOT]; #endif /* Ensure the destination has a higher/equal priority to us. */ if (unlikely(dest->tcbPriority < ksCurThread->tcbPriority)) { slowpath(SysCall); } /* Ensure that the endpoint has has grant rights so that we can * create the reply cap */ if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap))) { slowpath(SysCall); } #ifdef ARCH_ARM if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) { slowpath(SysCall); } #endif /* Ensure the original caller is in the current domain and can be scheduled directly. */ if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) { slowpath(SysCall); } /* * --- POINT OF NO RETURN --- * * At this stage, we have committed to performing the IPC. */ #ifdef ARCH_X86 /* Need to update NextIP in the calling thread */ setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2); #endif /* Dequeue the destination. */ endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext)); if (unlikely(dest->tcbEPNext)) { dest->tcbEPNext->tcbEPPrev = NULL; } else { endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle); } badge = cap_endpoint_cap_get_capEPBadge(ep_cap); /* Block sender */ thread_state_ptr_set_tsType_np(&ksCurThread->tcbState, ThreadState_BlockedOnReply); /* Get sender reply slot */ replySlot = TCB_PTR_CTE_PTR(ksCurThread, tcbReply); /* Get dest caller slot */ callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller); /* Insert reply cap */ cap_reply_cap_ptr_new_np(&callerSlot->cap, 0, TCB_REF(ksCurThread)); mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot)); mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( &replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1); fastpath_copy_mrs (length, ksCurThread, dest); /* Dest thread is set Running, but not queued. */ thread_state_ptr_set_tsType_np(&dest->tcbState, ThreadState_Running); switchToThread_fp(dest, cap_pd, stored_hw_asid); msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0)); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES benchmark_track_exit(); #endif fastpath_restore(badge, msgInfo, ksCurThread); }