word_t setMRs_fault(tcb_t *sender, tcb_t *receiver, word_t *receiveIPCBuffer)
{
switch (seL4_Fault_get_seL4_FaultType(sender->tcbFault)) {
case seL4_Fault_CapFault:
setMR(receiver, receiveIPCBuffer, seL4_CapFault_IP, getRestartPC(sender));
setMR(receiver, receiveIPCBuffer, seL4_CapFault_Addr,
seL4_Fault_CapFault_get_address(sender->tcbFault));
setMR(receiver, receiveIPCBuffer, seL4_CapFault_InRecvPhase,
seL4_Fault_CapFault_get_inReceivePhase(sender->tcbFault));
return setMRs_lookup_failure(receiver, receiveIPCBuffer,
sender->tcbLookupFailure, seL4_CapFault_LookupFailureType);
case seL4_Fault_UnknownSyscall: {
copyMRsFault(sender, receiver, MessageID_Syscall, n_syscallMessage,
receiveIPCBuffer);
return setMR(receiver, receiveIPCBuffer, n_syscallMessage,
seL4_Fault_UnknownSyscall_get_syscallNumber(sender->tcbFault));
}
case seL4_Fault_UserException: {
copyMRsFault(sender, receiver, MessageID_Exception,
n_exceptionMessage, receiveIPCBuffer);
setMR(receiver, receiveIPCBuffer, n_exceptionMessage,
seL4_Fault_UserException_get_number(sender->tcbFault));
return setMR(receiver, receiveIPCBuffer, n_exceptionMessage + 1u,
seL4_Fault_UserException_get_code(sender->tcbFault));
}
#ifdef CONFIG_HARDWARE_DEBUG_API
case seL4_Fault_DebugException: {
word_t reason = seL4_Fault_DebugException_get_exceptionReason(sender->tcbFault);
setMR(receiver, receiveIPCBuffer,
seL4_DebugException_FaultIP, getRestartPC(sender));
unsigned int ret = setMR(receiver, receiveIPCBuffer,
seL4_DebugException_ExceptionReason, reason);
if (reason != seL4_SingleStep && reason != seL4_SoftwareBreakRequest) {
ret = setMR(receiver, receiveIPCBuffer,
seL4_DebugException_TriggerAddress,
seL4_Fault_DebugException_get_breakpointAddress(sender->tcbFault));
/* Breakpoint messages also set a "breakpoint number" register. */
ret = setMR(receiver, receiveIPCBuffer,
seL4_DebugException_BreakpointNumber,
seL4_Fault_DebugException_get_breakpointNumber(sender->tcbFault));
}
return ret;
}
#endif /* CONFIG_HARDWARE_DEBUG_API */
default:
return Arch_setMRs_fault(sender, receiver, receiveIPCBuffer,
seL4_Fault_get_seL4_FaultType(sender->tcbFault));
}
}
void fastpath_reply_recv(word_t cptr, word_t msgInfo)
{
seL4_MessageInfo_t info;
cap_t ep_cap;
endpoint_t *ep_ptr;
word_t length;
cte_t *callerSlot;
cap_t callerCap;
tcb_t *caller;
word_t badge;
tcb_t *endpointTail;
word_t fault_type;
cap_t newVTable;
vspace_root_t *cap_pd;
pde_t stored_hw_asid;
dom_t dom;
/* Get message info and length */
info = messageInfoFromWord_raw(msgInfo);
length = seL4_MessageInfo_get_length(info);
fault_type = seL4_Fault_get_seL4_FaultType(NODE_STATE(ksCurThread)->tcbFault);
/* Check there's no extra caps, the length is ok and there's no
* saved fault. */
if (unlikely(fastpath_mi_check(msgInfo) ||
fault_type != seL4_Fault_NullFault)) {
slowpath(SysReplyRecv);
}
/* Lookup the cap */
ep_cap = lookup_fp(TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCTable)->cap,
cptr);
/* Check it's an endpoint */
if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) ||
!cap_endpoint_cap_get_capCanReceive(ep_cap))) {
slowpath(SysReplyRecv);
}
/* Check there is nothing waiting on the notification */
if (NODE_STATE(ksCurThread)->tcbBoundNotification &&
notification_ptr_get_state(NODE_STATE(ksCurThread)->tcbBoundNotification) == NtfnState_Active) {
slowpath(SysReplyRecv);
}
/* Get the endpoint address */
ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap));
/* Check that there's not a thread waiting to send */
if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) {
slowpath(SysReplyRecv);
}
/* Only reply if the reply cap is valid. */
callerSlot = TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCaller);
callerCap = callerSlot->cap;
if (unlikely(!fastpath_reply_cap_check(callerCap))) {
slowpath(SysReplyRecv);
}
/* Determine who the caller is. */
caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap));
/* ensure we are not single stepping the caller in ia32 */
#if defined(CONFIG_HARDWARE_DEBUG_API) && defined(CONFIG_ARCH_IA32)
if (caller->tcbArch.tcbContext.breakpointState.single_step_enabled) {
slowpath(SysReplyRecv);
}
#endif
/* Check that the caller has not faulted, in which case a fault
reply is generated instead. */
fault_type = seL4_Fault_get_seL4_FaultType(caller->tcbFault);
if (unlikely(fault_type != seL4_Fault_NullFault)) {
slowpath(SysReplyRecv);
}
/* Get destination thread.*/
newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap;
/* Get vspace root. */
cap_pd = cap_vtable_cap_get_vspace_root_fp(newVTable);
/* Ensure that the destination has a valid MMU. */
if (unlikely(! isValidVTableRoot_fp(newVTable))) {
slowpath(SysReplyRecv);
}
#ifdef CONFIG_ARCH_AARCH32
/* Get HWASID. */
stored_hw_asid = cap_pd[PD_ASID_SLOT];
#endif
#ifdef CONFIG_ARCH_X86_64
stored_hw_asid.words[0] = cap_pml4_cap_get_capPML4MappedASID(newVTable);
#endif
#ifdef CONFIG_ARCH_AARCH64
stored_hw_asid.words[0] = cap_page_global_directory_cap_get_capPGDMappedASID(newVTable);
#endif
#ifdef CONFIG_ARCH_RISCV
stored_hw_asid.words[0] = cap_page_table_cap_get_capPTMappedASID(newVTable);
#endif
/* Ensure the original caller can be scheduled directly. */
dom = maxDom ? ksCurDomain : 0;
if (unlikely(!isHighestPrio(dom, caller->tcbPriority))) {
slowpath(SysReplyRecv);
}
#ifdef CONFIG_ARCH_AARCH32
/* Ensure the HWASID is valid. */
if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) {
slowpath(SysReplyRecv);
}
#endif
/* Ensure the original caller is in the current domain and can be scheduled directly. */
if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) {
slowpath(SysReplyRecv);
}
#ifdef ENABLE_SMP_SUPPORT
/* Ensure both threads have the same affinity */
if (unlikely(NODE_STATE(ksCurThread)->tcbAffinity != caller->tcbAffinity)) {
slowpath(SysReplyRecv);
}
#endif /* ENABLE_SMP_SUPPORT */
/*
* --- POINT OF NO RETURN ---
*
* At this stage, we have committed to performing the IPC.
*/
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
ksKernelEntry.is_fastpath = true;
#endif
/* Set thread state to BlockedOnReceive */
thread_state_ptr_mset_blockingObject_tsType(
&NODE_STATE(ksCurThread)->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive);
thread_state_ptr_set_blockingIPCCanGrant(&NODE_STATE(ksCurThread)->tcbState,
cap_endpoint_cap_get_capCanGrant(ep_cap));;
/* Place the thread in the endpoint queue */
endpointTail = endpoint_ptr_get_epQueue_tail_fp(ep_ptr);
if (likely(!endpointTail)) {
NODE_STATE(ksCurThread)->tcbEPPrev = NULL;
NODE_STATE(ksCurThread)->tcbEPNext = NULL;
/* Set head/tail of queue and endpoint state. */
endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)));
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)),
EPState_Recv);
} else {
/* Append current thread onto the queue. */
endpointTail->tcbEPNext = NODE_STATE(ksCurThread);
NODE_STATE(ksCurThread)->tcbEPPrev = endpointTail;
NODE_STATE(ksCurThread)->tcbEPNext = NULL;
/* Update tail of queue. */
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(NODE_STATE(ksCurThread)),
EPState_Recv);
}
/* Delete the reply cap. */
mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
&CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode,
0, 1, 1);
callerSlot->cap = cap_null_cap_new();
callerSlot->cteMDBNode = nullMDBNode;
/* I know there's no fault, so straight to the transfer. */
/* Replies don't have a badge. */
badge = 0;
fastpath_copy_mrs(length, NODE_STATE(ksCurThread), caller);
/* Dest thread is set Running, but not queued. */
thread_state_ptr_set_tsType_np(&caller->tcbState,
ThreadState_Running);
switchToThread_fp(caller, cap_pd, stored_hw_asid);
msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0));
fastpath_restore(badge, msgInfo, NODE_STATE(ksCurThread));
}
void
#ifdef ARCH_X86
NORETURN
#endif
fastpath_call(word_t cptr, word_t msgInfo)
{
seL4_MessageInfo_t info;
cap_t ep_cap;
endpoint_t *ep_ptr;
word_t length;
tcb_t *dest;
word_t badge;
cte_t *replySlot, *callerSlot;
cap_t newVTable;
vspace_root_t *cap_pd;
pde_t stored_hw_asid;
word_t fault_type;
dom_t dom;
word_t replyCanGrant;
/* Get message info, length, and fault type. */
info = messageInfoFromWord_raw(msgInfo);
length = seL4_MessageInfo_get_length(info);
fault_type = seL4_Fault_get_seL4_FaultType(NODE_STATE(ksCurThread)->tcbFault);
/* Check there's no extra caps, the length is ok and there's no
* saved fault. */
if (unlikely(fastpath_mi_check(msgInfo) ||
fault_type != seL4_Fault_NullFault)) {
slowpath(SysCall);
}
/* Lookup the cap */
ep_cap = lookup_fp(TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbCTable)->cap, cptr);
/* Check it's an endpoint */
if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) ||
!cap_endpoint_cap_get_capCanSend(ep_cap))) {
slowpath(SysCall);
}
/* Get the endpoint address */
ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap));
/* Get the destination thread, which is only going to be valid
* if the endpoint is valid. */
dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr));
/* Check that there's a thread waiting to receive */
if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) {
slowpath(SysCall);
}
/* ensure we are not single stepping the destination in ia32 */
#if defined(CONFIG_HARDWARE_DEBUG_API) && defined(CONFIG_ARCH_IA32)
if (dest->tcbArch.tcbContext.breakpointState.single_step_enabled) {
slowpath(SysCall);
}
#endif
/* Get destination thread.*/
newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap;
/* Get vspace root. */
cap_pd = cap_vtable_cap_get_vspace_root_fp(newVTable);
/* Ensure that the destination has a valid VTable. */
if (unlikely(! isValidVTableRoot_fp(newVTable))) {
slowpath(SysCall);
}
#ifdef CONFIG_ARCH_AARCH32
/* Get HW ASID */
stored_hw_asid = cap_pd[PD_ASID_SLOT];
#endif
#ifdef CONFIG_ARCH_X86_64
/* borrow the stored_hw_asid for PCID */
stored_hw_asid.words[0] = cap_pml4_cap_get_capPML4MappedASID_fp(newVTable);
#endif
#ifdef CONFIG_ARCH_AARCH64
stored_hw_asid.words[0] = cap_page_global_directory_cap_get_capPGDMappedASID(newVTable);
#endif
#ifdef CONFIG_ARCH_RISCV
/* Get HW ASID */
stored_hw_asid.words[0] = cap_page_table_cap_get_capPTMappedASID(newVTable);
#endif
/* let gcc optimise this out for 1 domain */
dom = maxDom ? ksCurDomain : 0;
/* ensure only the idle thread or lower prio threads are present in the scheduler */
if (likely(dest->tcbPriority < NODE_STATE(ksCurThread->tcbPriority)) &&
!isHighestPrio(dom, dest->tcbPriority)) {
slowpath(SysCall);
}
/* Ensure that the endpoint has has grant or grant-reply rights so that we can
* create the reply cap */
if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap) &&
!cap_endpoint_cap_get_capCanGrantReply(ep_cap))) {
slowpath(SysCall);
}
#ifdef CONFIG_ARCH_AARCH32
if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) {
slowpath(SysCall);
}
#endif
/* Ensure the original caller is in the current domain and can be scheduled directly. */
if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) {
slowpath(SysCall);
}
#ifdef ENABLE_SMP_SUPPORT
/* Ensure both threads have the same affinity */
if (unlikely(NODE_STATE(ksCurThread)->tcbAffinity != dest->tcbAffinity)) {
slowpath(SysCall);
}
#endif /* ENABLE_SMP_SUPPORT */
/*
* --- POINT OF NO RETURN ---
*
* At this stage, we have committed to performing the IPC.
*/
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
ksKernelEntry.is_fastpath = true;
#endif
/* Dequeue the destination. */
endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext));
if (unlikely(dest->tcbEPNext)) {
dest->tcbEPNext->tcbEPPrev = NULL;
} else {
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle);
}
badge = cap_endpoint_cap_get_capEPBadge(ep_cap);
/* Block sender */
thread_state_ptr_set_tsType_np(&NODE_STATE(ksCurThread)->tcbState,
ThreadState_BlockedOnReply);
/* Get sender reply slot */
replySlot = TCB_PTR_CTE_PTR(NODE_STATE(ksCurThread), tcbReply);
/* Get dest caller slot */
callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller);
/* Insert reply cap */
replyCanGrant = thread_state_ptr_get_blockingIPCCanGrant(&dest->tcbState);;
cap_reply_cap_ptr_new_np(&callerSlot->cap, replyCanGrant, 0,
TCB_REF(NODE_STATE(ksCurThread)));
mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot));
mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
&replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1);
fastpath_copy_mrs(length, NODE_STATE(ksCurThread), dest);
/* Dest thread is set Running, but not queued. */
thread_state_ptr_set_tsType_np(&dest->tcbState,
ThreadState_Running);
switchToThread_fp(dest, cap_pd, stored_hw_asid);
msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0));
fastpath_restore(badge, msgInfo, NODE_STATE(ksCurThread));
}
void
receiveIPC(tcb_t *thread, cap_t cap, bool_t isBlocking)
{
endpoint_t *epptr;
notification_t *ntfnPtr;
/* Haskell error "receiveIPC: invalid cap" */
assert(cap_get_capType(cap) == cap_endpoint_cap);
epptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(cap));
/* Check for anything waiting in the notification */
ntfnPtr = thread->tcbBoundNotification;
if (ntfnPtr && notification_ptr_get_state(ntfnPtr) == NtfnState_Active) {
completeSignal(ntfnPtr, thread);
} else {
switch (endpoint_ptr_get_state(epptr)) {
case EPState_Idle:
case EPState_Recv: {
tcb_queue_t queue;
if (isBlocking) {
/* Set thread state to BlockedOnReceive */
thread_state_ptr_set_tsType(&thread->tcbState,
ThreadState_BlockedOnReceive);
thread_state_ptr_set_blockingObject(
&thread->tcbState, EP_REF(epptr));
scheduleTCB(thread);
/* Place calling thread in endpoint queue */
queue = ep_ptr_get_queue(epptr);
queue = tcbEPAppend(thread, queue);
endpoint_ptr_set_state(epptr, EPState_Recv);
ep_ptr_set_queue(epptr, queue);
} else {
doNBRecvFailedTransfer(thread);
}
break;
}
case EPState_Send: {
tcb_queue_t queue;
tcb_t *sender;
word_t badge;
bool_t canGrant;
bool_t do_call;
/* Get the head of the endpoint queue. */
queue = ep_ptr_get_queue(epptr);
sender = queue.head;
/* Haskell error "Send endpoint queue must not be empty" */
assert(sender);
/* Dequeue the first TCB */
queue = tcbEPDequeue(sender, queue);
ep_ptr_set_queue(epptr, queue);
if (!queue.head) {
endpoint_ptr_set_state(epptr, EPState_Idle);
}
/* Get sender IPC details */
badge = thread_state_ptr_get_blockingIPCBadge(&sender->tcbState);
canGrant =
thread_state_ptr_get_blockingIPCCanGrant(&sender->tcbState);
/* Do the transfer */
doIPCTransfer(sender, epptr, badge,
canGrant, thread);
do_call = thread_state_ptr_get_blockingIPCIsCall(&sender->tcbState);
if (do_call ||
seL4_Fault_get_seL4_FaultType(sender->tcbFault) != seL4_Fault_NullFault) {
if (canGrant) {
setupCallerCap(sender, thread);
} else {
setThreadState(sender, ThreadState_Inactive);
}
} else {
setThreadState(sender, ThreadState_Running);
switchIfRequiredTo(sender);
}
break;
}
}
}
}
bool_t handleFaultReply(tcb_t *receiver, tcb_t *sender)
{
/* These lookups are moved inward from doReplyTransfer */
seL4_MessageInfo_t tag = messageInfoFromWord(getRegister(sender, msgInfoRegister));
word_t label = seL4_MessageInfo_get_label(tag);
word_t length = seL4_MessageInfo_get_length(tag);
seL4_Fault_t fault = receiver->tcbFault;
switch (seL4_Fault_get_seL4_FaultType(fault)) {
case seL4_Fault_CapFault:
return true;
case seL4_Fault_UnknownSyscall:
copyMRsFaultReply(sender, receiver, MessageID_Syscall, MIN(length, n_syscallMessage));
return (label == 0);
case seL4_Fault_UserException:
copyMRsFaultReply(sender, receiver, MessageID_Exception, MIN(length, n_exceptionMessage));
return (label == 0);
#ifdef CONFIG_HARDWARE_DEBUG_API
case seL4_Fault_DebugException: {
word_t n_instrs;
if (seL4_Fault_DebugException_get_exceptionReason(fault) != seL4_SingleStep) {
/* Only single-step replies are required to set message registers.
*/
return (label == 0);
}
if (length < DEBUG_REPLY_N_EXPECTED_REGISTERS) {
/* A single-step reply doesn't mean much if it isn't composed of the bp
* number and number of instructions to skip. But even if both aren't
* set, we can still allow the thread to continue because replying
* should uniformly resume thread execution, based on the general seL4
* API model.
*
* If it was single-step, but no reply registers were set, just
* default to skipping 1 and continuing.
*
* On x86, bp_num actually doesn't matter for single-stepping
* because single-stepping doesn't use a hardware register -- it
* uses EFLAGS.TF.
*/
n_instrs = 1;
} else {
/* If the reply had all expected registers set, proceed as normal */
n_instrs = getRegister(sender, msgRegisters[0]);
}
syscall_error_t res;
res = Arch_decodeConfigureSingleStepping(receiver, 0, n_instrs, true);
if (res.type != seL4_NoError) {
return false;
};
configureSingleStepping(receiver, 0, n_instrs, true);
/* Replying will always resume the thread: the only variant behaviour
* is whether or not the thread will be resumed with stepping still
* enabled.
*/
return (label == 0);
}
#endif
default:
return Arch_handleFaultReply(receiver, sender, seL4_Fault_get_seL4_FaultType(fault));
}
}
