void
doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot)
{
assert(thread_state_get_tsType(receiver->tcbState) ==
ThreadState_BlockedOnReply);
if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) {
doIPCTransfer(sender, NULL, 0, true, receiver, false);
setThreadState(receiver, ThreadState_Running);
attemptSwitchTo(receiver);
} else {
bool_t restart;
restart = handleFaultReply(receiver, sender);
fault_null_fault_ptr_new(&receiver->tcbFault);
if (restart) {
setThreadState(receiver, ThreadState_Restart);
attemptSwitchTo(receiver);
} else {
setThreadState(receiver, ThreadState_Inactive);
}
}
if (cap_reply_cap_get_capInCDT(slot->cap)) {
cte_t *replySlot = TCB_PTR_CTE_PTR(receiver, tcbReply);
assert(cap_get_capType(replySlot->cap) == cap_reply_cap);
assert(cap_reply_cap_get_capInCDT(replySlot->cap));
cdtRemove(replySlot);
cdtRemove(slot);
slot->cap = cap_null_cap_new();
replySlot->cap = cap_reply_cap_new(false, true, TCB_REF(NULL));
} else {
deleteCallerCap(sender);
}
}
void
doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot)
{
assert(thread_state_get_tsType(receiver->tcbState) ==
ThreadState_BlockedOnReply);
if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) {
doIPCTransfer(sender, NULL, 0, true, receiver, false);
/** GHOSTUPD: "(True, gs_set_assn cteDeleteOne_'proc (ucast cap_reply_cap))" */
cteDeleteOne(slot);
setThreadState(receiver, ThreadState_Running);
attemptSwitchTo(receiver);
} else {
bool_t restart;
/** GHOSTUPD: "(True, gs_set_assn cteDeleteOne_'proc (ucast cap_reply_cap))" */
cteDeleteOne(slot);
restart = handleFaultReply(receiver, sender);
fault_null_fault_ptr_new(&receiver->tcbFault);
if (restart) {
setThreadState(receiver, ThreadState_Restart);
attemptSwitchTo(receiver);
} else {
setThreadState(receiver, ThreadState_Inactive);
}
}
}
void
doReplyTransfer(tcb_t *sender, tcb_t *receiver, cte_t *slot)
{
assert(thread_state_get_tsType(receiver->tcbState) ==
ThreadState_BlockedOnReply);
if (likely(fault_get_faultType(receiver->tcbFault) == fault_null_fault)) {
doIPCTransfer(sender, NULL, 0, true, receiver, false);
setThreadState(receiver, ThreadState_Running);
attemptSwitchTo(receiver);
} else {
bool_t restart;
restart = handleFaultReply(receiver, sender);
fault_null_fault_ptr_new(&receiver->tcbFault);
if (restart) {
setThreadState(receiver, ThreadState_Restart);
attemptSwitchTo(receiver);
} else {
setThreadState(receiver, ThreadState_Inactive);
}
}
finaliseCap(slot->cap, true, true);
slot->cap = cap_null_cap_new();
}
void
doFaultTransfer(word_t badge, tcb_t *sender, tcb_t *receiver,
word_t *receiverIPCBuffer)
{
unsigned int sent;
message_info_t msgInfo;
sent = setMRs_fault(sender, receiver, receiverIPCBuffer);
msgInfo = message_info_new(
fault_get_faultType(sender->tcbFault), 0, 0, sent);
setRegister(receiver, msgInfoRegister, wordFromMessageInfo(msgInfo));
setRegister(receiver, badgeRegister, badge);
}
void
doIPCTransfer(tcb_t *sender, endpoint_t *endpoint, word_t badge,
bool_t grant, tcb_t *receiver, bool_t diminish)
{
void *receiveBuffer, *sendBuffer;
receiveBuffer = lookupIPCBuffer(true, receiver);
if (likely(!fault_get_faultType(sender->tcbFault) != fault_null_fault)) {
sendBuffer = lookupIPCBuffer(false, sender);
doNormalTransfer(sender, sendBuffer, endpoint, badge, grant,
receiver, receiveBuffer, diminish);
} else {
doFaultTransfer(badge, sender, receiver, receiveBuffer);
}
}
void
receiveIPC(tcb_t *thread, cap_t cap)
{
endpoint_t *epptr;
bool_t diminish;
async_endpoint_t *aepptr;
/* Haskell error "receiveIPC: invalid cap" */
assert(cap_get_capType(cap) == cap_endpoint_cap);
//printf("\n;;;;;;;;;In function receiveIPC;;;;;;;\n");
epptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(cap));
diminish = !cap_endpoint_cap_get_capCanSend(cap);
/* Check for anything waiting in the async endpoint*/
aepptr = thread->boundAsyncEndpoint;
if (aepptr && async_endpoint_ptr_get_state(aepptr) == AEPState_Active)
{
completeAsyncIPC(aepptr, thread);
}
else
{
switch (endpoint_ptr_get_state(epptr)) {
case EPState_Idle:
//printf("in case idle\n");
case EPState_Recv: {
tcb_queue_t queue;
//printf("in case recv\n");
/* Set thread state to BlockedOnReceive */
thread_state_ptr_set_tsType(&thread->tcbState,
ThreadState_BlockedOnReceive);
thread_state_ptr_set_blockingIPCEndpoint(
&thread->tcbState, EP_REF(epptr));
thread_state_ptr_set_blockingIPCDiminishCaps(
&thread->tcbState, diminish);
scheduleTCB(thread);
/* Place calling thread in endpoint queue */
queue = ep_ptr_get_queue(epptr);
queue = tcbEPAppend(thread, queue);
endpoint_ptr_set_state(epptr, EPState_Recv);
ep_ptr_set_queue(epptr, queue);
break;
}
case EPState_Send: {
tcb_queue_t queue;
tcb_t *sender;
word_t badge;
bool_t canGrant;
bool_t do_call;
//printf("in case send\n");
/* Get the head of the endpoint queue. */
queue = ep_ptr_get_queue(epptr);
sender = queue.head;
/* Haskell error "Send endpoint queue must not be empty" */
assert(sender);
/* Dequeue the first TCB */
queue = tcbEPDequeue(sender, queue);
ep_ptr_set_queue(epptr, queue);
if (!queue.head) {
endpoint_ptr_set_state(epptr, EPState_Idle);
}
/* Get sender IPC details */
badge = thread_state_ptr_get_blockingIPCBadge(&sender->tcbState);
canGrant =
thread_state_ptr_get_blockingIPCCanGrant(&sender->tcbState);
/* Do the transfer */
doIPCTransfer(sender, epptr, badge,
canGrant, thread, diminish);
do_call = thread_state_ptr_get_blockingIPCIsCall(&sender->tcbState);
if (do_call ||
fault_get_faultType(sender->tcbFault) != fault_null_fault) {
if (canGrant && !diminish) {
setupCallerCap(sender, thread);
} else {
setThreadState(sender, ThreadState_Inactive);
}
} else {
setThreadState(sender, ThreadState_Running);
switchIfRequiredTo(sender);
}
break;
}
}
}
}
bool_t
handleFaultReply(tcb_t *receiver, tcb_t *sender)
{
message_info_t tag;
word_t label;
fault_t fault;
unsigned int length;
/* These lookups are moved inward from doReplyTransfer */
tag = messageInfoFromWord(getRegister(sender, msgInfoRegister));
label = message_info_get_msgLabel(tag);
length = message_info_get_msgLength(tag);
fault = receiver->tcbFault;
switch (fault_get_faultType(fault)) {
case fault_cap_fault:
return true;
case fault_vm_fault:
return true;
case fault_unknown_syscall: {
unsigned int i;
register_t r;
word_t v;
word_t *sendBuf;
sendBuf = lookupIPCBuffer(false, sender);
/* Assumes n_syscallMessage > n_msgRegisters */
for (i = 0; i < length && i < n_msgRegisters; i++) {
r = syscallMessage[i];
v = getRegister(sender, msgRegisters[i]);
setRegister(receiver, r, sanitiseRegister(r, v));
}
if (sendBuf) {
for (; i < length && i < n_syscallMessage; i++) {
r = syscallMessage[i];
v = sendBuf[i + 1];
setRegister(receiver, r, sanitiseRegister(r, v));
}
}
}
return (label == 0);
case fault_user_exception: {
unsigned int i;
register_t r;
word_t v;
/* Assumes n_exceptionMessage <= n_msgRegisters */
for (i = 0; i < length && i < n_exceptionMessage; i++) {
r = exceptionMessage[i];
v = getRegister(sender, msgRegisters[i]);
setRegister(receiver, r, sanitiseRegister(r, v));
}
}
return (label == 0);
default:
fail("Invalid fault");
}
}
unsigned int setMRs_fault(tcb_t *sender, tcb_t* receiver, word_t *receiveIPCBuffer)
{
assert(n_msgRegisters == 2);
switch (fault_get_faultType(sender->tcbFault)) {
case fault_cap_fault:
setRegister(receiver, msgRegisters[0], getRestartPC(sender));
setRegister(receiver, msgRegisters[1],
fault_cap_fault_get_address(sender->tcbFault));
if (!receiveIPCBuffer) {
return n_msgRegisters;
}
receiveIPCBuffer[2 + 1] =
fault_cap_fault_get_inReceivePhase(sender->tcbFault);
return setMRs_lookup_failure(receiver, receiveIPCBuffer, sender->tcbLookupFailure, 3);
case fault_vm_fault:
setRegister(receiver, msgRegisters[0], getRestartPC(sender));
setRegister(receiver, msgRegisters[1],
fault_vm_fault_get_address(sender->tcbFault));
if (!receiveIPCBuffer) {
return n_msgRegisters;
}
receiveIPCBuffer[2 + 1] =
fault_vm_fault_get_instructionFault(sender->tcbFault);
receiveIPCBuffer[3 + 1] = fault_vm_fault_get_FSR(sender->tcbFault);
return 4;
case fault_unknown_syscall: {
unsigned int i;
for (i = 0; i < n_msgRegisters; i++) {
setRegister(receiver, msgRegisters[i],
getRegister(sender, syscallMessage[i]));
}
if (receiveIPCBuffer) {
for (; i < n_syscallMessage; i++) {
receiveIPCBuffer[i + 1] =
getRegister(sender, syscallMessage[i]);
}
receiveIPCBuffer[i + 1] =
fault_unknown_syscall_get_syscallNumber(sender->tcbFault);
return n_syscallMessage + 1;
} else {
return n_msgRegisters;
}
}
case fault_user_exception: {
unsigned int i;
for (i = 0; i < n_msgRegisters; i++) {
setRegister(receiver, msgRegisters[i],
getRegister(sender, exceptionMessage[i]));
}
if (receiveIPCBuffer) {
for (; i < n_exceptionMessage; i++) {
receiveIPCBuffer[i + 1] =
getRegister(sender, exceptionMessage[i]);
}
receiveIPCBuffer[n_exceptionMessage + 1] =
fault_user_exception_get_number(sender->tcbFault);
receiveIPCBuffer[n_exceptionMessage + 2] =
fault_user_exception_get_code(sender->tcbFault);
return n_exceptionMessage + 2;
} else {
return n_msgRegisters;
}
}
default:
fail("Invalid fault");
}
}
void
fastpath_reply_recv(word_t cptr, word_t msgInfo)
{
seL4_MessageInfo_t info;
cap_t ep_cap;
endpoint_t *ep_ptr;
word_t length;
cte_t *callerSlot;
cap_t callerCap;
tcb_t *caller;
word_t badge;
tcb_t *endpointTail;
word_t fault_type;
cap_t newVTable;
pde_t *cap_pd;
pde_t stored_hw_asid;
/* Get message info and length */
info = messageInfoFromWord_raw(msgInfo);
length = seL4_MessageInfo_get_length(info);
fault_type = fault_get_faultType(ksCurThread->tcbFault);
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
ksKernelEntry.path = Entry_Syscall;
ksKernelEntry.syscall_no = SysReplyRecv;
ksKernelEntry.cap_type = cap_endpoint_cap;
ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info);
ksKernelEntry.is_fastpath = true;
benchmark_track_start();
#endif
#ifdef CONFIG_BENCHMARK_TRACK_UTILISATION
benchmark_utilisation_kentry_stamp();
#endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */
/* Check there's no extra caps, the length is ok and there's no
* saved fault. */
if (unlikely(fastpath_mi_check(msgInfo) ||
fault_type != fault_null_fault)) {
slowpath(SysReplyRecv);
}
/* Lookup the cap */
ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap,
cptr);
/* Check it's an endpoint */
if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) ||
!cap_endpoint_cap_get_capCanReceive(ep_cap))) {
slowpath(SysReplyRecv);
}
/* Check there is nothing waiting on the notification */
if (ksCurThread->tcbBoundNotification &&
notification_ptr_get_state(ksCurThread->tcbBoundNotification) == NtfnState_Active) {
slowpath(SysReplyRecv);
}
/* Get the endpoint address */
ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap));
/* Check that there's not a thread waiting to send */
if (unlikely(endpoint_ptr_get_state(ep_ptr) == EPState_Send)) {
slowpath(SysReplyRecv);
}
/* Only reply if the reply cap is valid. */
callerSlot = TCB_PTR_CTE_PTR(ksCurThread, tcbCaller);
callerCap = callerSlot->cap;
if (unlikely(!fastpath_reply_cap_check(callerCap))) {
slowpath(SysReplyRecv);
}
/* Determine who the caller is. */
caller = TCB_PTR(cap_reply_cap_get_capTCBPtr(callerCap));
/* Check that the caller has not faulted, in which case a fault
reply is generated instead. */
fault_type = fault_get_faultType(caller->tcbFault);
if (unlikely(fault_type != fault_null_fault)) {
slowpath(SysReplyRecv);
}
/* Get destination thread.*/
newVTable = TCB_PTR_CTE_PTR(caller, tcbVTable)->cap;
/* Get vspace root. */
#if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING)
cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable));
#else
cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable));
#endif
/* Ensure that the destination has a valid MMU. */
if (unlikely(! isValidVTableRoot_fp (newVTable))) {
slowpath(SysReplyRecv);
}
#ifdef ARCH_ARM
/* Get HWASID. */
stored_hw_asid = cap_pd[PD_ASID_SLOT];
#endif
/* Ensure the original caller can be scheduled directly. */
if (unlikely(caller->tcbPriority < ksCurThread->tcbPriority)) {
slowpath(SysReplyRecv);
}
#ifdef ARCH_ARM
/* Ensure the HWASID is valid. */
if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) {
slowpath(SysReplyRecv);
}
#endif
/* Ensure the original caller is in the current domain and can be scheduled directly. */
if (unlikely(caller->tcbDomain != ksCurDomain && maxDom)) {
slowpath(SysReplyRecv);
}
/*
* --- POINT OF NO RETURN ---
*
* At this stage, we have committed to performing the IPC.
*/
#ifdef ARCH_X86
/* Need to update NextIP in the calling thread */
setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2);
#endif
/* Set thread state to BlockedOnReceive */
thread_state_ptr_mset_blockingObject_tsType(
&ksCurThread->tcbState, (word_t)ep_ptr, ThreadState_BlockedOnReceive);
/* Place the thread in the endpoint queue */
endpointTail = TCB_PTR(endpoint_ptr_get_epQueue_tail(ep_ptr));
if (likely(!endpointTail)) {
ksCurThread->tcbEPPrev = NULL;
ksCurThread->tcbEPNext = NULL;
/* Set head/tail of queue and endpoint state. */
endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(ksCurThread));
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread),
EPState_Recv);
} else {
/* Append current thread onto the queue. */
endpointTail->tcbEPNext = ksCurThread;
ksCurThread->tcbEPPrev = endpointTail;
ksCurThread->tcbEPNext = NULL;
/* Update tail of queue. */
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, TCB_REF(ksCurThread),
EPState_Recv);
}
/* Delete the reply cap. */
mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
&CTE_PTR(mdb_node_get_mdbPrev(callerSlot->cteMDBNode))->cteMDBNode,
0, 1, 1);
callerSlot->cap = cap_null_cap_new();
callerSlot->cteMDBNode = nullMDBNode;
/* I know there's no fault, so straight to the transfer. */
/* Replies don't have a badge. */
badge = 0;
fastpath_copy_mrs (length, ksCurThread, caller);
/* Dest thread is set Running, but not queued. */
thread_state_ptr_set_tsType_np(&caller->tcbState,
ThreadState_Running);
switchToThread_fp(caller, cap_pd, stored_hw_asid);
msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0));
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
benchmark_track_exit();
#endif
fastpath_restore(badge, msgInfo, ksCurThread);
}
void
#ifdef ARCH_X86
NORETURN
#endif
fastpath_call(word_t cptr, word_t msgInfo)
{
seL4_MessageInfo_t info;
cap_t ep_cap;
endpoint_t *ep_ptr;
word_t length;
tcb_t *dest;
word_t badge;
cte_t *replySlot, *callerSlot;
cap_t newVTable;
pde_t *cap_pd;
pde_t stored_hw_asid;
word_t fault_type;
/* Get message info, length, and fault type. */
info = messageInfoFromWord_raw(msgInfo);
length = seL4_MessageInfo_get_length(info);
fault_type = fault_get_faultType(ksCurThread->tcbFault);
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
ksKernelEntry.path = Entry_Syscall;
ksKernelEntry.syscall_no = SysCall;
ksKernelEntry.cap_type = cap_endpoint_cap;
ksKernelEntry.invocation_tag = seL4_MessageInfo_get_label(info);
ksKernelEntry.is_fastpath = true;
benchmark_track_start();
#endif
#ifdef CONFIG_BENCHMARK_TRACK_UTILISATION
benchmark_utilisation_kentry_stamp();
#endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */
/* Check there's no extra caps, the length is ok and there's no
* saved fault. */
if (unlikely(fastpath_mi_check(msgInfo) ||
fault_type != fault_null_fault)) {
slowpath(SysCall);
}
/* Lookup the cap */
ep_cap = lookup_fp(TCB_PTR_CTE_PTR(ksCurThread, tcbCTable)->cap, cptr);
/* Check it's an endpoint */
if (unlikely(!cap_capType_equals(ep_cap, cap_endpoint_cap) ||
!cap_endpoint_cap_get_capCanSend(ep_cap))) {
slowpath(SysCall);
}
/* Get the endpoint address */
ep_ptr = EP_PTR(cap_endpoint_cap_get_capEPPtr(ep_cap));
/* Get the destination thread, which is only going to be valid
* if the endpoint is valid. */
dest = TCB_PTR(endpoint_ptr_get_epQueue_head(ep_ptr));
/* Check that there's a thread waiting to receive */
if (unlikely(endpoint_ptr_get_state(ep_ptr) != EPState_Recv)) {
slowpath(SysCall);
}
/* Get destination thread.*/
newVTable = TCB_PTR_CTE_PTR(dest, tcbVTable)->cap;
/* Get vspace root. */
#if defined(ARCH_ARM) || !defined(CONFIG_PAE_PAGING)
cap_pd = PDE_PTR(cap_page_directory_cap_get_capPDBasePtr(newVTable));
#else
cap_pd = PDE_PTR(cap_pdpt_cap_get_capPDPTBasePtr(newVTable));
#endif
/* Ensure that the destination has a valid VTable. */
if (unlikely(! isValidVTableRoot_fp(newVTable))) {
slowpath(SysCall);
}
#ifdef ARCH_ARM
/* Get HW ASID */
stored_hw_asid = cap_pd[PD_ASID_SLOT];
#endif
/* Ensure the destination has a higher/equal priority to us. */
if (unlikely(dest->tcbPriority < ksCurThread->tcbPriority)) {
slowpath(SysCall);
}
/* Ensure that the endpoint has has grant rights so that we can
* create the reply cap */
if (unlikely(!cap_endpoint_cap_get_capCanGrant(ep_cap))) {
slowpath(SysCall);
}
#ifdef ARCH_ARM
if (unlikely(!pde_pde_invalid_get_stored_asid_valid(stored_hw_asid))) {
slowpath(SysCall);
}
#endif
/* Ensure the original caller is in the current domain and can be scheduled directly. */
if (unlikely(dest->tcbDomain != ksCurDomain && maxDom)) {
slowpath(SysCall);
}
/*
* --- POINT OF NO RETURN ---
*
* At this stage, we have committed to performing the IPC.
*/
#ifdef ARCH_X86
/* Need to update NextIP in the calling thread */
setRegister(ksCurThread, NextIP, getRegister(ksCurThread, NextIP) + 2);
#endif
/* Dequeue the destination. */
endpoint_ptr_set_epQueue_head_np(ep_ptr, TCB_REF(dest->tcbEPNext));
if (unlikely(dest->tcbEPNext)) {
dest->tcbEPNext->tcbEPPrev = NULL;
} else {
endpoint_ptr_mset_epQueue_tail_state(ep_ptr, 0, EPState_Idle);
}
badge = cap_endpoint_cap_get_capEPBadge(ep_cap);
/* Block sender */
thread_state_ptr_set_tsType_np(&ksCurThread->tcbState,
ThreadState_BlockedOnReply);
/* Get sender reply slot */
replySlot = TCB_PTR_CTE_PTR(ksCurThread, tcbReply);
/* Get dest caller slot */
callerSlot = TCB_PTR_CTE_PTR(dest, tcbCaller);
/* Insert reply cap */
cap_reply_cap_ptr_new_np(&callerSlot->cap, 0, TCB_REF(ksCurThread));
mdb_node_ptr_set_mdbPrev_np(&callerSlot->cteMDBNode, CTE_REF(replySlot));
mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
&replySlot->cteMDBNode, CTE_REF(callerSlot), 1, 1);
fastpath_copy_mrs (length, ksCurThread, dest);
/* Dest thread is set Running, but not queued. */
thread_state_ptr_set_tsType_np(&dest->tcbState,
ThreadState_Running);
switchToThread_fp(dest, cap_pd, stored_hw_asid);
msgInfo = wordFromMessageInfo(seL4_MessageInfo_set_capsUnwrapped(info, 0));
#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES
benchmark_track_exit();
#endif
fastpath_restore(badge, msgInfo, ksCurThread);
}
