mach_msg_return_t
mach_msg_send_from_kernel_proper(
mach_msg_header_t *msg,
mach_msg_size_t send_size)
{
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
mr = ipc_kmsg_copyin_from_kernel(kmsg);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return mr;
}
mr = ipc_kmsg_send(kmsg,
MACH_SEND_KERNEL_DEFAULT,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_destroy(kmsg);
}
return mr;
}
/* lifted from ipc_mig.c:mach_msg_send_from_kernel_proper() */
static mach_msg_return_t
mach_msg_send_from_remote_kernel(mach_msg_header_t *msg,
mach_msg_size_t send_size,
mach_node_t node)
{
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
mr = ipc_kmsg_copyin_from_kernel(kmsg);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return mr;
}
kmsg->ikm_node = node; // node that needs to receive message ack
mr = ipc_kmsg_send(kmsg,
MACH_SEND_KERNEL_DEFAULT,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_destroy(kmsg);
}
return mr;
}
kern_return_t
exception_parse_reply(ipc_kmsg_t kmsg)
{
mig_reply_header_t *msg =
(mig_reply_header_t *) &kmsg->ikm_header;
kern_return_t kr;
if ((msg->Head.msgh_bits !=
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0)) ||
(msg->Head.msgh_size != sizeof *msg) ||
(msg->Head.msgh_id != MACH_EXCEPTION_REPLY_ID) ||
(BAD_TYPECHECK(&msg->RetCodeType, &exc_RetCode_proto))) {
/*
* Bozo user sent us a misformatted reply.
*/
kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
ipc_kmsg_destroy(kmsg);
return MIG_REPLY_MISMATCH;
}
kr = msg->RetCode;
if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
(ikm_cache() == IKM_NULL))
ikm_cache() = kmsg;
else
ikm_free(kmsg);
return kr;
}
mach_msg_return_t
mach_msg_send_from_kernel_with_options(
mach_msg_header_t *msg,
mach_msg_size_t send_size,
mach_msg_option_t option,
mach_msg_timeout_t timeout_val)
{
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS) {
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
return mr;
}
mr = ipc_kmsg_copyin_from_kernel(kmsg);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
return mr;
}
/*
* Until we are sure of its effects, we are disabling
* importance donation from the kernel-side of user
* threads in importance-donating tasks - unless the
* option to force importance donation is passed in,
* or the thread's SEND_IMPORTANCE option has been set.
* (11938665 & 23925818)
*/
if (current_thread()->options & TH_OPT_SEND_IMPORTANCE)
option &= ~MACH_SEND_NOIMPORTANCE;
else if ((option & MACH_SEND_IMPORTANCE) == 0)
option |= MACH_SEND_NOIMPORTANCE;
mr = ipc_kmsg_send(kmsg, option, timeout_val);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_destroy(kmsg);
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
}
return mr;
}
mach_msg_return_t
mach_msg_send_from_kernel_proper(
mach_msg_header_t *msg,
mach_msg_size_t send_size)
{
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS) {
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
return mr;
}
mr = ipc_kmsg_copyin_from_kernel(kmsg);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
return mr;
}
/*
* respect the thread's SEND_IMPORTANCE option to force importance
* donation from the kernel-side of user threads
* (11938665 & 23925818)
*/
mach_msg_option_t option = MACH_SEND_KERNEL_DEFAULT;
if (current_thread()->options & TH_OPT_SEND_IMPORTANCE)
option &= ~MACH_SEND_NOIMPORTANCE;
mr = ipc_kmsg_send(kmsg, option, MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_destroy(kmsg);
KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
}
return mr;
}
mach_msg_return_t
mach_msg_send_from_kernel_with_options(
mach_msg_header_t *msg,
mach_msg_size_t send_size,
mach_msg_option_t option,
mach_msg_timeout_t timeout_val)
{
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
mr = ipc_kmsg_copyin_from_kernel(kmsg);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return mr;
}
#if 11938665
/*
* Until we are sure of its effects, we are disabling
* importance donation from the kernel-side of user
* threads in importance-donating tasks - unless the
* option to force importance donation is passed in.
*/
if ((option & MACH_SEND_IMPORTANCE) == 0)
option |= MACH_SEND_NOIMPORTANCE;
#endif
mr = ipc_kmsg_send(kmsg, option, timeout_val);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_destroy(kmsg);
}
return mr;
}
mach_msg_return_t
mach_msg_rpc_from_kernel_body(
mach_msg_header_t *msg,
mach_msg_size_t send_size,
mach_msg_size_t rcv_size,
#if !IKM_SUPPORT_LEGACY
__unused
#endif
boolean_t legacy)
{
thread_t self = current_thread();
ipc_port_t reply;
ipc_kmsg_t kmsg;
mach_port_seqno_t seqno;
mach_msg_return_t mr;
assert(msg->msgh_local_port == MACH_PORT_NULL);
mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
reply = self->ith_rpc_reply;
if (reply == IP_NULL) {
reply = ipc_port_alloc_reply();
if ((reply == IP_NULL) ||
(self->ith_rpc_reply != IP_NULL))
panic("mach_msg_rpc_from_kernel");
self->ith_rpc_reply = reply;
}
/* insert send-once right for the reply port */
kmsg->ikm_header->msgh_local_port = reply;
kmsg->ikm_header->msgh_bits |=
MACH_MSGH_BITS(0, MACH_MSG_TYPE_MAKE_SEND_ONCE);
#if IKM_SUPPORT_LEGACY
if(legacy)
mr = ipc_kmsg_copyin_from_kernel_legacy(kmsg);
else
mr = ipc_kmsg_copyin_from_kernel(kmsg);
#else
mr = ipc_kmsg_copyin_from_kernel(kmsg);
#endif
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return mr;
}
mr = ipc_kmsg_send(kmsg,
MACH_SEND_KERNEL_DEFAULT,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_destroy(kmsg);
return mr;
}
for (;;) {
ipc_mqueue_t mqueue;
assert(reply->ip_pset_count == 0);
assert(ip_active(reply));
/* JMM - why this check? */
if (!self->active) {
ipc_port_dealloc_reply(reply);
self->ith_rpc_reply = IP_NULL;
return MACH_RCV_INTERRUPTED;
}
self->ith_continuation = (void (*)(mach_msg_return_t))0;
mqueue = &reply->ip_messages;
ipc_mqueue_receive(mqueue,
MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
THREAD_INTERRUPTIBLE);
mr = self->ith_state;
kmsg = self->ith_kmsg;
seqno = self->ith_seqno;
if (mr == MACH_MSG_SUCCESS)
{
break;
}
assert(mr == MACH_RCV_INTERRUPTED);
assert(reply == self->ith_rpc_reply);
if (self->handlers) {
ipc_port_dealloc_reply(reply);
self->ith_rpc_reply = IP_NULL;
return(mr);
}
}
/*
* Check to see how much of the message/trailer can be received.
* We chose the maximum trailer that will fit, since we don't
* have options telling us which trailer elements the caller needed.
*/
if (rcv_size >= kmsg->ikm_header->msgh_size) {
mach_msg_format_0_trailer_t *trailer = (mach_msg_format_0_trailer_t *)
((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size);
if (rcv_size >= kmsg->ikm_header->msgh_size + MAX_TRAILER_SIZE) {
/* Enough room for a maximum trailer */
trailer->msgh_trailer_size = MAX_TRAILER_SIZE;
}
else if (rcv_size < kmsg->ikm_header->msgh_size +
trailer->msgh_trailer_size) {
/* no room for even the basic (default) trailer */
trailer->msgh_trailer_size = 0;
}
assert(trailer->msgh_trailer_type == MACH_MSG_TRAILER_FORMAT_0);
rcv_size = kmsg->ikm_header->msgh_size + trailer->msgh_trailer_size;
mr = MACH_MSG_SUCCESS;
} else {
mr = MACH_RCV_TOO_LARGE;
}
/*
* We want to preserve rights and memory in reply!
* We don't have to put them anywhere; just leave them
* as they are.
*/
#if IKM_SUPPORT_LEGACY
if(legacy)
ipc_kmsg_copyout_to_kernel_legacy(kmsg, ipc_space_reply);
else
ipc_kmsg_copyout_to_kernel(kmsg, ipc_space_reply);
#else
ipc_kmsg_copyout_to_kernel(kmsg, ipc_space_reply);
#endif
ipc_kmsg_put_to_kernel(msg, kmsg, rcv_size);
return mr;
}
ipc_kmsg_t
ipc_kobject_server(
ipc_kmsg_t request,
mach_msg_option_t __unused option)
{
mach_msg_size_t reply_size;
ipc_kmsg_t reply;
kern_return_t kr;
ipc_port_t *destp;
ipc_port_t replyp = IPC_PORT_NULL;
mach_msg_format_0_trailer_t *trailer;
mig_hash_t *ptr;
task_t task = TASK_NULL;
uint32_t exec_token;
boolean_t exec_token_changed = FALSE;
/*
* Find out corresponding mig_hash entry if any
*/
{
int key = request->ikm_header->msgh_id;
unsigned int i = (unsigned int)MIG_HASH(key);
int max_iter = mig_table_max_displ;
do {
ptr = &mig_buckets[i++ % MAX_MIG_ENTRIES];
} while (key != ptr->num && ptr->num && --max_iter);
if (!ptr->routine || key != ptr->num) {
ptr = (mig_hash_t *)0;
reply_size = mig_reply_size;
} else {
reply_size = ptr->size;
#if MACH_COUNTER
ptr->callcount++;
#endif
}
}
/* round up for trailer size */
reply_size += MAX_TRAILER_SIZE;
reply = ipc_kmsg_alloc(reply_size);
if (reply == IKM_NULL) {
printf("ipc_kobject_server: dropping request\n");
ipc_kmsg_trace_send(request, option);
ipc_kmsg_destroy(request);
return IKM_NULL;
}
/*
* Initialize reply message.
*/
{
#define InP ((mach_msg_header_t *) request->ikm_header)
#define OutP ((mig_reply_error_t *) reply->ikm_header)
/*
* MIG should really assure no data leakage -
* but until it does, pessimistically zero the
* whole reply buffer.
*/
bzero((void *)OutP, reply_size);
OutP->NDR = NDR_record;
OutP->Head.msgh_size = sizeof(mig_reply_error_t);
OutP->Head.msgh_bits =
MACH_MSGH_BITS_SET(MACH_MSGH_BITS_LOCAL(InP->msgh_bits), 0, 0, 0);
OutP->Head.msgh_remote_port = InP->msgh_local_port;
OutP->Head.msgh_local_port = MACH_PORT_NULL;
OutP->Head.msgh_voucher_port = MACH_PORT_NULL;
OutP->Head.msgh_id = InP->msgh_id + 100;
#undef InP
#undef OutP
}
/*
* Find the routine to call, and call it
* to perform the kernel function
*/
ipc_kmsg_trace_send(request, option);
{
if (ptr) {
/*
* Check if the port is a task port, if its a task port then
* snapshot the task exec token before the mig routine call.
*/
ipc_port_t port = request->ikm_header->msgh_remote_port;
if (IP_VALID(port) && ip_kotype(port) == IKOT_TASK) {
task = convert_port_to_task_with_exec_token(port, &exec_token);
}
(*ptr->routine)(request->ikm_header, reply->ikm_header);
/* Check if the exec token changed during the mig routine */
if (task != TASK_NULL) {
if (exec_token != task->exec_token) {
exec_token_changed = TRUE;
}
task_deallocate(task);
}
kernel_task->messages_received++;
}
else {
if (!ipc_kobject_notify(request->ikm_header, reply->ikm_header)){
#if DEVELOPMENT || DEBUG
printf("ipc_kobject_server: bogus kernel message, id=%d\n",
request->ikm_header->msgh_id);
#endif /* DEVELOPMENT || DEBUG */
_MIG_MSGID_INVALID(request->ikm_header->msgh_id);
((mig_reply_error_t *) reply->ikm_header)->RetCode
= MIG_BAD_ID;
}
else
kernel_task->messages_received++;
}
kernel_task->messages_sent++;
}
/*
* Destroy destination. The following code differs from
* ipc_object_destroy in that we release the send-once
* right instead of generating a send-once notification
* (which would bring us here again, creating a loop).
* It also differs in that we only expect send or
* send-once rights, never receive rights.
*
* We set msgh_remote_port to IP_NULL so that the kmsg
* destroy routines don't try to destroy the port twice.
*/
destp = (ipc_port_t *) &request->ikm_header->msgh_remote_port;
switch (MACH_MSGH_BITS_REMOTE(request->ikm_header->msgh_bits)) {
case MACH_MSG_TYPE_PORT_SEND:
ipc_port_release_send(*destp);
break;
case MACH_MSG_TYPE_PORT_SEND_ONCE:
ipc_port_release_sonce(*destp);
break;
default:
panic("ipc_kobject_server: strange destination rights");
}
*destp = IP_NULL;
/*
* Destroy voucher. The kernel MIG servers never take ownership
* of vouchers sent in messages. Swallow any such rights here.
*/
if (IP_VALID(request->ikm_voucher)) {
assert(MACH_MSG_TYPE_PORT_SEND ==
MACH_MSGH_BITS_VOUCHER(request->ikm_header->msgh_bits));
ipc_port_release_send(request->ikm_voucher);
request->ikm_voucher = IP_NULL;
}
if (!(reply->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) &&
((mig_reply_error_t *) reply->ikm_header)->RetCode != KERN_SUCCESS)
kr = ((mig_reply_error_t *) reply->ikm_header)->RetCode;
else
kr = KERN_SUCCESS;
if ((kr == KERN_SUCCESS) || (kr == MIG_NO_REPLY)) {
/*
* The server function is responsible for the contents
* of the message. The reply port right is moved
* to the reply message, and we have deallocated
* the destination port right, so we just need
* to free the kmsg.
*/
ipc_kmsg_free(request);
} else {
/*
* The message contents of the request are intact.
* Destroy everthing except the reply port right,
* which is needed in the reply message.
*/
request->ikm_header->msgh_local_port = MACH_PORT_NULL;
ipc_kmsg_destroy(request);
}
replyp = (ipc_port_t)reply->ikm_header->msgh_remote_port;
if (kr == MIG_NO_REPLY) {
/*
* The server function will send a reply message
* using the reply port right, which it has saved.
*/
ipc_kmsg_free(reply);
return IKM_NULL;
} else if (!IP_VALID(replyp)) {
/*
* Can't queue the reply message if the destination
* (the reply port) isn't valid.
*/
ipc_kmsg_destroy(reply);
return IKM_NULL;
} else if (replyp->ip_receiver == ipc_space_kernel) {
/*
* Don't send replies to kobject kernel ports
*/
#if DEVELOPMENT || DEBUG
printf("%s: refusing to send reply to kobject %d port (id:%d)\n",
__func__, ip_kotype(replyp),
request->ikm_header->msgh_id);
#endif /* DEVELOPMENT || DEBUG */
ipc_kmsg_destroy(reply);
return IKM_NULL;
}
/* Fail the MIG call if the task exec token changed during the call */
if (kr == KERN_SUCCESS && exec_token_changed) {
/*
* Create a new reply msg with error and destroy the old reply msg.
*/
ipc_kmsg_t new_reply = ipc_kmsg_alloc(reply_size);
if (new_reply == IKM_NULL) {
printf("ipc_kobject_server: dropping request\n");
ipc_kmsg_destroy(reply);
return IKM_NULL;
}
/*
* Initialize the new reply message.
*/
{
#define OutP_new ((mig_reply_error_t *) new_reply->ikm_header)
#define OutP_old ((mig_reply_error_t *) reply->ikm_header)
bzero((void *)OutP_new, reply_size);
OutP_new->NDR = OutP_old->NDR;
OutP_new->Head.msgh_size = sizeof(mig_reply_error_t);
OutP_new->Head.msgh_bits = OutP_old->Head.msgh_bits & ~MACH_MSGH_BITS_COMPLEX;
OutP_new->Head.msgh_remote_port = OutP_old->Head.msgh_remote_port;
OutP_new->Head.msgh_local_port = MACH_PORT_NULL;
OutP_new->Head.msgh_voucher_port = MACH_PORT_NULL;
OutP_new->Head.msgh_id = OutP_old->Head.msgh_id;
/* Set the error as KERN_INVALID_TASK */
OutP_new->RetCode = KERN_INVALID_TASK;
#undef OutP_new
#undef OutP_old
}
/*
* Destroy everything in reply except the reply port right,
* which is needed in the new reply message.
*/
reply->ikm_header->msgh_remote_port = MACH_PORT_NULL;
ipc_kmsg_destroy(reply);
reply = new_reply;
}
trailer = (mach_msg_format_0_trailer_t *)
((vm_offset_t)reply->ikm_header + (int)reply->ikm_header->msgh_size);
trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
return reply;
}
ipc_kmsg_t
ipc_kobject_server(
ipc_kmsg_t request)
{
mach_msg_size_t reply_size;
ipc_kmsg_t reply;
kern_return_t kr;
mig_routine_t routine;
ipc_port_t *destp;
mach_msg_format_0_trailer_t *trailer;
register mig_hash_t *ptr;
#if MACH_RT
boolean_t reply_rt;
#endif /* MACH_RT */
unsigned int th;
/* Only fetch current thread if ETAP is configured */
ETAP_DATA_LOAD(th, current_thread());
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_BEGIN,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
/*
* Find out corresponding mig_hash entry if any
*/
{
register int key = request->ikm_header.msgh_id;
register int i = MIG_HASH(key);
register int max_iter = mig_table_max_displ;
do
ptr = &mig_buckets[i++ % MAX_MIG_ENTRIES];
while (key != ptr->num && ptr->num && --max_iter);
if (!ptr->routine || key != ptr->num) {
ptr = (mig_hash_t *)0;
reply_size = mig_reply_size;
} else {
reply_size = ptr->size;
#if MACH_COUNTER
ptr->callcount++;
#endif
}
}
/* round up for ikm_cache; overhead is added by ikm_alloc */
if ((reply_size += MAX_TRAILER_SIZE) < IKM_SAVED_MSG_SIZE)
reply_size = IKM_SAVED_MSG_SIZE;
#if MACH_RT
reply_rt =
IP_VALID((ipc_port_t)request->ikm_header.msgh_local_port) ?
IP_RT((ipc_port_t)request->ikm_header.msgh_local_port) :
FALSE;
if (reply_rt)
reply = ikm_rtalloc(reply_size);
else
#endif /* MACH_RT */
reply = ikm_alloc(reply_size);
if (reply == IKM_NULL) {
printf("ipc_kobject_server: dropping request\n");
ipc_kmsg_destroy(request);
return IKM_NULL;
}
ikm_init(reply, reply_size);
#if DIPC
reply->ikm_handle = HANDLE_NULL;
#endif /* DIPC */
/*
* Initialize reply message.
*/
{
#define InP ((mach_msg_header_t *) &request->ikm_header)
#define OutP ((mig_reply_error_t *) &reply->ikm_header)
OutP->NDR = NDR_record;
OutP->Head.msgh_size = sizeof(mig_reply_error_t);
OutP->Head.msgh_bits =
MACH_MSGH_BITS(MACH_MSGH_BITS_LOCAL(InP->msgh_bits), 0);
OutP->Head.msgh_remote_port = InP->msgh_local_port;
OutP->Head.msgh_local_port = MACH_PORT_NULL;
OutP->Head.msgh_id = InP->msgh_id + 100;
#if MACH_RT
if (reply_rt)
KMSG_MARK_RT(reply);
#endif /* MACH_RT */
#undef InP
#undef OutP
}
/*
* Find the routine to call, and call it
* to perform the kernel function
*/
{
if (ptr)
(*ptr->routine)(&request->ikm_header, &reply->ikm_header);
else {
if (!ipc_kobject_notify(&request->ikm_header, &reply->ikm_header)){
#if MACH_IPC_TEST
printf("ipc_kobject_server: bogus kernel message, id=%d\n",
request->ikm_header.msgh_id);
#endif /* MACH_IPC_TEST */
((mig_reply_error_t *) &reply->ikm_header)->RetCode
= MIG_BAD_ID;
}
}
}
/*
* Destroy destination. The following code differs from
* ipc_object_destroy in that we release the send-once
* right instead of generating a send-once notification
* (which would bring us here again, creating a loop).
* It also differs in that we only expect send or
* send-once rights, never receive rights.
*
* We set msgh_remote_port to IP_NULL so that the kmsg
* destroy routines don't try to destroy the port twice.
*/
destp = (ipc_port_t *) &request->ikm_header.msgh_remote_port;
switch (MACH_MSGH_BITS_REMOTE(request->ikm_header.msgh_bits)) {
case MACH_MSG_TYPE_PORT_SEND:
ipc_port_release_send(*destp);
break;
case MACH_MSG_TYPE_PORT_SEND_ONCE:
ipc_port_release_sonce(*destp);
break;
default:
panic("ipc_object_destroy: strange destination rights");
}
*destp = IP_NULL;
if (!(reply->ikm_header.msgh_bits & MACH_MSGH_BITS_COMPLEX) &&
((mig_reply_error_t *) &reply->ikm_header)->RetCode != KERN_SUCCESS)
kr = ((mig_reply_error_t *) &reply->ikm_header)->RetCode;
else
kr = KERN_SUCCESS;
if ((kr == KERN_SUCCESS) || (kr == MIG_NO_REPLY)) {
/*
* The server function is responsible for the contents
* of the message. The reply port right is moved
* to the reply message, and we have deallocated
* the destination port right, so we just need
* to free the kmsg.
*/
/*
* Like ipc_kmsg_put, but without the copyout. Also,
* messages to the kernel will never have been allocated
* from the rt_zone.
*/
ikm_check_initialized(request, request->ikm_size);
if (request->ikm_size != IKM_SAVED_KMSG_SIZE ||
!ikm_cache_put (request)) {
ikm_free(request);
}
} else {
/*
* The message contents of the request are intact.
* Destroy everthing except the reply port right,
* which is needed in the reply message.
*/
request->ikm_header.msgh_local_port = MACH_PORT_NULL;
ipc_kmsg_destroy(request);
}
if (kr == MIG_NO_REPLY) {
/*
* The server function will send a reply message
* using the reply port right, which it has saved.
*/
ikm_free(reply);
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_END,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
return IKM_NULL;
} else if (!IP_VALID((ipc_port_t)reply->ikm_header.msgh_remote_port)) {
/*
* Can't queue the reply message if the destination
* (the reply port) isn't valid.
*/
ipc_kmsg_destroy(reply);
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_END,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
return IKM_NULL;
}
trailer = (mach_msg_format_0_trailer_t *)
((vm_offset_t)&reply->ikm_header + (int)reply->ikm_header.msgh_size);
trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_END,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
return reply;
}
void
ipc_port_destroy(
ipc_port_t port)
{
ipc_port_t pdrequest, nsrequest;
ipc_mqueue_t mqueue;
ipc_kmsg_queue_t kmqueue;
ipc_kmsg_t kmsg;
ipc_thread_t sender;
ipc_port_request_t dnrequests;
thread_pool_t thread_pool;
assert(ip_active(port));
/* port->ip_receiver_name is garbage */
/* port->ip_receiver/port->ip_destination is garbage */
assert(io_otype((ipc_object_t)port) == IOT_PORT);
assert(port->ip_pset == IPS_NULL);
assert(port->ip_mscount == 0);
assert(port->ip_seqno == 0);
/* first check for a backup port */
pdrequest = port->ip_pdrequest;
if (pdrequest != IP_NULL) {
/* we assume the ref for pdrequest */
port->ip_pdrequest = IP_NULL;
/* make port be in limbo */
port->ip_receiver_name = MACH_PORT_NAME_NULL;
port->ip_destination = IP_NULL;
ip_unlock(port);
if (!ipc_port_check_circularity(port, pdrequest)) {
/* consumes our refs for port and pdrequest */
ipc_notify_port_destroyed(pdrequest, port);
return;
} else {
/* consume pdrequest and destroy port */
ipc_port_release_sonce(pdrequest);
}
ip_lock(port);
assert(ip_active(port));
assert(port->ip_pset == IPS_NULL);
assert(port->ip_mscount == 0);
assert(port->ip_seqno == 0);
assert(port->ip_pdrequest == IP_NULL);
assert(port->ip_receiver_name == MACH_PORT_NAME_NULL);
assert(port->ip_destination == IP_NULL);
/* fall through and destroy the port */
}
/*
* rouse all blocked senders
*
* This must be done with the port locked, because
* ipc_mqueue_send can play with the ip_blocked queue
* of a dead port.
*/
while ((sender = ipc_thread_dequeue(&port->ip_blocked)) != ITH_NULL) {
sender->ith_state = MACH_MSG_SUCCESS;
thread_go(sender);
}
/* once port is dead, we don't need to keep it locked */
port->ip_object.io_bits &= ~IO_BITS_ACTIVE;
port->ip_timestamp = ipc_port_timestamp();
/* save for later */
dnrequests = port->ip_dnrequests;
port->ip_dnrequests = IPR_NULL;
ip_unlock(port);
/* wakeup any threads waiting on this pool port for an activation */
if ((thread_pool = &port->ip_thread_pool) != THREAD_POOL_NULL)
thread_pool_wakeup(thread_pool);
/* throw away no-senders request */
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL)
ipc_notify_send_once(nsrequest); /* consumes ref */
/* destroy any queued messages */
mqueue = &port->ip_messages;
kmqueue = &mqueue->imq_messages;
while ((kmsg = ipc_kmsg_dequeue(kmqueue)) != IKM_NULL) {
assert(kmsg->ikm_header->msgh_remote_port ==
(mach_port_t) port);
port->ip_msgcount--;
ipc_port_release(port);
kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
ipc_kmsg_destroy(kmsg);
}
/* generate dead-name notifications */
if (dnrequests != IPR_NULL) {
ipc_port_dnnotify(port, dnrequests);
}
if (ip_kotype(port) != IKOT_NONE)
ipc_kobject_destroy(port);
/* XXXX Perhaps should verify that ip_thread_pool is empty! */
ipc_port_release(port); /* consume caller's ref */
}
ipc_kmsg_t
ipc_kobject_server(
ipc_kmsg_t request)
{
mach_msg_size_t reply_size;
ipc_kmsg_t reply;
kern_return_t kr;
ipc_port_t *destp;
mach_msg_format_0_trailer_t *trailer;
register mig_hash_t *ptr;
/*
* Find out corresponding mig_hash entry if any
*/
{
register int key = request->ikm_header->msgh_id;
register int i = MIG_HASH(key);
register int max_iter = mig_table_max_displ;
do
ptr = &mig_buckets[i++ % MAX_MIG_ENTRIES];
while (key != ptr->num && ptr->num && --max_iter);
if (!ptr->routine || key != ptr->num) {
ptr = (mig_hash_t *)0;
reply_size = mig_reply_size;
} else {
reply_size = ptr->size;
#if MACH_COUNTER
ptr->callcount++;
#endif
}
}
/* round up for trailer size */
reply_size += MAX_TRAILER_SIZE;
reply = ipc_kmsg_alloc(reply_size);
if (reply == IKM_NULL) {
printf("ipc_kobject_server: dropping request\n");
ipc_kmsg_destroy(request);
return IKM_NULL;
}
/*
* Initialize reply message.
*/
{
#define InP ((mach_msg_header_t *) request->ikm_header)
#define OutP ((mig_reply_error_t *) reply->ikm_header)
/*
* MIG should really assure no data leakage -
* but until it does, pessimistically zero the
* whole reply buffer.
*/
bzero((void *)OutP, reply_size);
OutP->NDR = NDR_record;
OutP->Head.msgh_size = sizeof(mig_reply_error_t);
OutP->Head.msgh_bits =
MACH_MSGH_BITS_SET(MACH_MSGH_BITS_LOCAL(InP->msgh_bits), 0, 0, 0);
OutP->Head.msgh_remote_port = InP->msgh_local_port;
OutP->Head.msgh_local_port = MACH_PORT_NULL;
OutP->Head.msgh_voucher_port = MACH_PORT_NULL;
OutP->Head.msgh_id = InP->msgh_id + 100;
#undef InP
#undef OutP
}
/*
* Find the routine to call, and call it
* to perform the kernel function
*/
{
if (ptr) {
(*ptr->routine)(request->ikm_header, reply->ikm_header);
kernel_task->messages_received++;
}
else {
if (!ipc_kobject_notify(request->ikm_header, reply->ikm_header)){
#if MACH_IPC_TEST
printf("ipc_kobject_server: bogus kernel message, id=%d\n",
request->ikm_header->msgh_id);
#endif /* MACH_IPC_TEST */
_MIG_MSGID_INVALID(request->ikm_header->msgh_id);
((mig_reply_error_t *) reply->ikm_header)->RetCode
= MIG_BAD_ID;
}
else
kernel_task->messages_received++;
}
kernel_task->messages_sent++;
}
/*
* Destroy destination. The following code differs from
* ipc_object_destroy in that we release the send-once
* right instead of generating a send-once notification
* (which would bring us here again, creating a loop).
* It also differs in that we only expect send or
* send-once rights, never receive rights.
*
* We set msgh_remote_port to IP_NULL so that the kmsg
* destroy routines don't try to destroy the port twice.
*/
destp = (ipc_port_t *) &request->ikm_header->msgh_remote_port;
switch (MACH_MSGH_BITS_REMOTE(request->ikm_header->msgh_bits)) {
case MACH_MSG_TYPE_PORT_SEND:
ipc_port_release_send(*destp);
break;
case MACH_MSG_TYPE_PORT_SEND_ONCE:
ipc_port_release_sonce(*destp);
break;
default:
panic("ipc_kobject_server: strange destination rights");
}
*destp = IP_NULL;
/*
* Destroy voucher. The kernel MIG servers never take ownership
* of vouchers sent in messages. Swallow any such rights here.
*/
if (IP_VALID(request->ikm_voucher)) {
assert(MACH_MSG_TYPE_PORT_SEND ==
MACH_MSGH_BITS_VOUCHER(request->ikm_header->msgh_bits));
ipc_port_release_send(request->ikm_voucher);
request->ikm_voucher = IP_NULL;
}
if (!(reply->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) &&
((mig_reply_error_t *) reply->ikm_header)->RetCode != KERN_SUCCESS)
kr = ((mig_reply_error_t *) reply->ikm_header)->RetCode;
else
kr = KERN_SUCCESS;
if ((kr == KERN_SUCCESS) || (kr == MIG_NO_REPLY)) {
/*
* The server function is responsible for the contents
* of the message. The reply port right is moved
* to the reply message, and we have deallocated
* the destination port right, so we just need
* to free the kmsg.
*/
ipc_kmsg_free(request);
} else {
/*
* The message contents of the request are intact.
* Destroy everthing except the reply port right,
* which is needed in the reply message.
*/
request->ikm_header->msgh_local_port = MACH_PORT_NULL;
ipc_kmsg_destroy(request);
}
if (kr == MIG_NO_REPLY) {
/*
* The server function will send a reply message
* using the reply port right, which it has saved.
*/
ipc_kmsg_free(reply);
return IKM_NULL;
} else if (!IP_VALID((ipc_port_t)reply->ikm_header->msgh_remote_port)) {
/*
* Can't queue the reply message if the destination
* (the reply port) isn't valid.
*/
ipc_kmsg_destroy(reply);
return IKM_NULL;
}
trailer = (mach_msg_format_0_trailer_t *)
((vm_offset_t)reply->ikm_header + (int)reply->ikm_header->msgh_size);
trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
return reply;
}
void
exception_raise(
ipc_port_t dest_port,
ipc_port_t thread_port,
ipc_port_t task_port,
integer_t _exception,
integer_t code,
integer_t subcode)
{
ipc_thread_t self = current_thread();
ipc_thread_t receiver;
ipc_port_t reply_port;
ipc_mqueue_t dest_mqueue;
ipc_mqueue_t reply_mqueue;
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
assert(IP_VALID(dest_port));
/*
* We will eventually need a message buffer.
* Grab the buffer now, while nothing is locked.
* This buffer will get handed to the exception server,
* and it will give the buffer back with its reply.
*/
kmsg = ikm_cache();
if (kmsg != IKM_NULL) {
ikm_cache() = IKM_NULL;
ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
} else {
kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
if (kmsg == IKM_NULL)
panic("exception_raise");
ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
}
/*
* We need a reply port for the RPC.
* Check first for a cached port.
*/
ith_lock(self);
assert(self->ith_self != IP_NULL);
reply_port = self->ith_rpc_reply;
if (reply_port == IP_NULL) {
ith_unlock(self);
reply_port = ipc_port_alloc_reply();
ith_lock(self);
if ((reply_port == IP_NULL) ||
(self->ith_rpc_reply != IP_NULL))
panic("exception_raise");
self->ith_rpc_reply = reply_port;
}
ip_lock(reply_port);
assert(ip_active(reply_port));
ith_unlock(self);
/*
* Make a naked send-once right for the reply port,
* to hand to the exception server.
* Make an extra reference for the reply port,
* to receive on. This protects us against
* mach_msg_abort_rpc.
*/
reply_port->ip_sorights++;
ip_reference(reply_port);
ip_reference(reply_port);
self->ith_port = reply_port;
reply_mqueue = &reply_port->ip_messages;
imq_lock(reply_mqueue);
assert(ipc_kmsg_queue_empty(&reply_mqueue->imq_messages));
ip_unlock(reply_port);
/*
* Make sure we can queue to the destination port.
*/
if (!ip_lock_try(dest_port)) {
imq_unlock(reply_mqueue);
goto slow_exception_raise;
}
if (!ip_active(dest_port) ||
(dest_port->ip_receiver == ipc_space_kernel)) {
imq_unlock(reply_mqueue);
ip_unlock(dest_port);
goto slow_exception_raise;
}
/*
* Find the destination message queue.
*/
{
ipc_pset_t dest_pset;
dest_pset = dest_port->ip_pset;
if (dest_pset == IPS_NULL)
dest_mqueue = &dest_port->ip_messages;
else
dest_mqueue = &dest_pset->ips_messages;
}
if (!imq_lock_try(dest_mqueue)) {
imq_unlock(reply_mqueue);
ip_unlock(dest_port);
goto slow_exception_raise;
}
/*
* Safe to unlock dest_port, because we hold
* dest_mqueue locked. We never bother changing
* dest_port->ip_msgcount.
*/
ip_unlock(dest_port);
receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
if ((receiver == ITH_NULL) ||
!((receiver->swap_func == (void (*)()) mach_msg_continue) ||
((receiver->swap_func ==
(void (*)()) mach_msg_receive_continue) &&
(sizeof(struct mach_exception) <= receiver->ith_msize) &&
((receiver->ith_option & MACH_RCV_NOTIFY) == 0))) ||
!thread_handoff(self, exception_raise_continue, receiver)) {
imq_unlock(reply_mqueue);
imq_unlock(dest_mqueue);
goto slow_exception_raise;
}
counter(c_exception_raise_block++);
assert(current_thread() == receiver);
/*
* We need to finish preparing self for its
* time asleep in reply_mqueue. self is left
* holding the extra ref for reply_port.
*/
ipc_thread_enqueue_macro(&reply_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(reply_mqueue);
/*
* Finish extracting receiver from dest_mqueue.
*/
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
imq_unlock(dest_mqueue);
/*
* Release the receiver's reference for his object.
*/
{
ipc_object_t object = receiver->ith_object;
io_lock(object);
io_release(object);
io_check_unlock(object);
}
{
struct mach_exception *exc =
(struct mach_exception *) &kmsg->ikm_header;
ipc_space_t space = receiver->task->itk_space;
/*
* We are running as the receiver now. We hold
* the following resources, which must be consumed:
* kmsg, send-once right for reply_port
* send rights for dest_port, thread_port, task_port
* Synthesize a kmsg for copyout to the receiver.
*/
exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PORT_SEND) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_size = sizeof *exc;
/* exc->Head.msgh_remote_port later */
/* exc->Head.msgh_local_port later */
exc->Head.msgh_seqno = 0;
exc->Head.msgh_id = MACH_EXCEPTION_ID;
exc->threadType = exc_port_proto;
/* exc->thread later */
exc->taskType = exc_port_proto;
/* exc->task later */
exc->exceptionType = exc_code_proto;
exc->exception = _exception;
exc->codeType = exc_code_proto;
exc->code = code;
exc->subcodeType = exc_code_proto;
exc->subcode = subcode;
/*
* Check that the receiver can handle the message.
*/
if (receiver->ith_rcv_size < sizeof(struct mach_exception)) {
/*
* ipc_kmsg_destroy is a handy way to consume
* the resources we hold, but it requires setup.
*/
exc->Head.msgh_bits =
(MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
exc->thread = (mach_port_t) thread_port;
exc->task = (mach_port_t) task_port;
ipc_kmsg_destroy(kmsg);
thread_syscall_return(MACH_RCV_TOO_LARGE);
/*NOTREACHED*/
}
is_write_lock(space);
assert(space->is_active);
/*
* To do an atomic copyout, need simultaneous
* locks on both ports and the space.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
abort_copyout:
ip_unlock(dest_port);
is_write_unlock(space);
/*
* Oh well, we have to do the header the slow way.
* First make it look like it's in-transit.
*/
exc->Head.msgh_bits =
(MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
mr = ipc_kmsg_copyout_header(&exc->Head, space,
MACH_PORT_NULL);
if (mr == MACH_MSG_SUCCESS)
goto copyout_body;
/*
* Ack! Prepare for ipc_kmsg_copyout_dest.
* It will consume thread_port and task_port.
*/
exc->thread = (mach_port_t) thread_port;
exc->task = (mach_port_t) task_port;
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(receiver->ith_msg, kmsg,
sizeof(mach_msg_header_t));
thread_syscall_return(mr);
/*NOTREACHED*/
}
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
goto abort_copyout;
}
assert(reply_port->ip_sorights > 0);
ip_unlock(reply_port);
{
kern_return_t kr;
ipc_entry_t entry;
kr = ipc_entry_get (space, &exc->Head.msgh_remote_port, &entry);
if (kr)
goto abort_copyout;
{
mach_port_gen_t gen;
assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
gen = entry->ie_bits + IE_BITS_GEN_ONE;
/* optimized ipc_right_copyout */
entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
}
entry->ie_object = (ipc_object_t) reply_port;
is_write_unlock(space);
}
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_srights > 0);
ip_release(dest_port);
exc->Head.msgh_local_port =
((dest_port->ip_receiver == space) ?
dest_port->ip_receiver_name : MACH_PORT_NULL);
if ((--dest_port->ip_srights == 0) &&
(dest_port->ip_nsrequest != IP_NULL)) {
ipc_port_t nsrequest;
mach_port_mscount_t mscount;
/* a rather rare case */
nsrequest = dest_port->ip_nsrequest;
mscount = dest_port->ip_mscount;
dest_port->ip_nsrequest = IP_NULL;
ip_unlock(dest_port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(dest_port);
copyout_body:
/*
* Optimized version of ipc_kmsg_copyout_body,
* to handle the two ports in the body.
*/
mr = (ipc_kmsg_copyout_object(space, (ipc_object_t) thread_port,
MACH_MSG_TYPE_PORT_SEND, &exc->thread) |
ipc_kmsg_copyout_object(space, (ipc_object_t) task_port,
MACH_MSG_TYPE_PORT_SEND, &exc->task));
if (mr != MACH_MSG_SUCCESS) {
(void) ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr | MACH_RCV_BODY_ERROR);
/*NOTREACHED*/
}
}
/*
* Optimized version of ipc_kmsg_put.
* We must check ikm_cache after copyoutmsg.
*/
ikm_check_initialized(kmsg, kmsg->ikm_size);
assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
if (copyoutmsg(&kmsg->ikm_header, receiver->ith_msg,
sizeof(struct mach_exception)) ||
(ikm_cache() != IKM_NULL)) {
mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
ikm_cache() = kmsg;
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
#ifndef __GNUC__
return; /* help for the compiler */
#endif
slow_exception_raise: {
struct mach_exception *exc =
(struct mach_exception *) &kmsg->ikm_header;
ipc_kmsg_t reply_kmsg;
mach_port_seqno_t reply_seqno;
exception_raise_misses++;
/*
* We hold the following resources, which must be consumed:
* kmsg, send-once right and ref for reply_port
* send rights for dest_port, thread_port, task_port
* Synthesize a kmsg to send.
*/
exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_size = sizeof *exc;
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
exc->Head.msgh_seqno = 0;
exc->Head.msgh_id = MACH_EXCEPTION_ID;
exc->threadType = exc_port_proto;
exc->thread = (mach_port_t) thread_port;
exc->taskType = exc_port_proto;
exc->task = (mach_port_t) task_port;
exc->exceptionType = exc_code_proto;
exc->exception = _exception;
exc->codeType = exc_code_proto;
exc->code = code;
exc->subcodeType = exc_code_proto;
exc->subcode = subcode;
ipc_mqueue_send_always(kmsg);
/*
* We are left with a ref for reply_port,
* which we use to receive the reply message.
*/
ip_lock(reply_port);
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
exception_raise_continue_slow(MACH_RCV_PORT_DIED, IKM_NULL, /*dummy*/0);
/*NOTREACHED*/
}
imq_lock(reply_mqueue);
ip_unlock(reply_port);
mr = ipc_mqueue_receive(reply_mqueue, MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, exception_raise_continue,
&reply_kmsg, &reply_seqno);
/* reply_mqueue is unlocked */
exception_raise_continue_slow(mr, reply_kmsg, reply_seqno);
/*NOTREACHED*/
}
}
mach_msg_return_t
ipc_mqueue_send(
ipc_kmsg_t kmsg,
mach_msg_option_t option,
mach_msg_timeout_t time_out)
{
ipc_port_t port;
port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
assert(IP_VALID(port));
ip_lock(port);
if (port->ip_receiver == ipc_space_kernel) {
ipc_kmsg_t reply;
/*
* We can check ip_receiver == ipc_space_kernel
* before checking that the port is active because
* ipc_port_dealloc_kernel clears ip_receiver
* before destroying a kernel port.
*/
assert(ip_active(port));
ip_unlock(port);
reply = ipc_kobject_server(kmsg);
if (reply != IKM_NULL)
ipc_mqueue_send_always(reply);
return MACH_MSG_SUCCESS;
}
for (;;) {
ipc_thread_t self;
/*
* Can't deliver to a dead port.
* However, we can pretend it got sent
* and was then immediately destroyed.
*/
if (!ip_active(port)) {
/*
* We can't let ipc_kmsg_destroy deallocate
* the port right, because we might end up
* in an infinite loop trying to deliver
* a send-once notification.
*/
ip_release(port);
ip_check_unlock(port);
kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
ipc_kmsg_destroy(kmsg);
return MACH_MSG_SUCCESS;
}
/*
* Don't block if:
* 1) We're under the queue limit.
* 2) Caller used the MACH_SEND_ALWAYS internal option.
* 3) Message is sent to a send-once right.
*/
if ((port->ip_msgcount < port->ip_qlimit) ||
(option & MACH_SEND_ALWAYS) ||
(MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
MACH_MSG_TYPE_PORT_SEND_ONCE))
break;
/* must block waiting for queue to clear */
self = current_thread();
if (option & MACH_SEND_TIMEOUT) {
if (time_out == 0) {
ip_unlock(port);
return MACH_SEND_TIMED_OUT;
}
thread_will_wait_with_timeout(self, time_out);
} else
thread_will_wait(self);
ipc_thread_enqueue(&port->ip_blocked, self);
self->ith_state = MACH_SEND_IN_PROGRESS;
ip_unlock(port);
counter(c_ipc_mqueue_send_block++);
thread_block((void (*)(void)) 0);
ip_lock(port);
/* why did we wake up? */
if (self->ith_state == MACH_MSG_SUCCESS)
continue;
assert(self->ith_state == MACH_SEND_IN_PROGRESS);
/* take ourselves off blocked queue */
ipc_thread_rmqueue(&port->ip_blocked, self);
/*
* Thread wakeup-reason field tells us why
* the wait was interrupted.
*/
switch (self->ith_wait_result) {
case THREAD_INTERRUPTED:
/* send was interrupted - give up */
ip_unlock(port);
return MACH_SEND_INTERRUPTED;
case THREAD_TIMED_OUT:
/* timeout expired */
assert(option & MACH_SEND_TIMEOUT);
time_out = 0;
break;
case THREAD_RESTART:
default:
#if MACH_ASSERT
assert(!"ipc_mqueue_send");
#else
panic("ipc_mqueue_send");
#endif
}
}
if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
ip_unlock(port);
/* don't allow the creation of a circular loop */
ipc_kmsg_destroy(kmsg);
return MACH_MSG_SUCCESS;
}
{
ipc_mqueue_t mqueue;
ipc_pset_t pset;
ipc_thread_t receiver;
ipc_thread_queue_t receivers;
port->ip_msgcount++;
assert(port->ip_msgcount > 0);
pset = port->ip_pset;
if (pset == IPS_NULL)
mqueue = &port->ip_messages;
else
mqueue = &pset->ips_messages;
imq_lock(mqueue);
receivers = &mqueue->imq_threads;
/*
* Can unlock the port now that the msg queue is locked
* and we know the port is active. While the msg queue
* is locked, we have control of the kmsg, so the ref in
* it for the port is still good. If the msg queue is in
* a set (dead or alive), then we're OK because the port
* is still a member of the set and the set won't go away
* until the port is taken out, which tries to lock the
* set's msg queue to remove the port's msgs.
*/
ip_unlock(port);
/* check for a receiver for the message */
for (;;) {
receiver = ipc_thread_queue_first(receivers);
if (receiver == ITH_NULL) {
/* no receivers; queue kmsg */
ipc_kmsg_enqueue_macro(&mqueue->imq_messages, kmsg);
imq_unlock(mqueue);
break;
}
ipc_thread_rmqueue_first_macro(receivers, receiver);
assert(ipc_kmsg_queue_empty(&mqueue->imq_messages));
if (kmsg->ikm_header.msgh_size <= receiver->ith_msize) {
/* got a successful receiver */
receiver->ith_state = MACH_MSG_SUCCESS;
receiver->ith_kmsg = kmsg;
receiver->ith_seqno = port->ip_seqno++;
imq_unlock(mqueue);
thread_go(receiver);
break;
}
receiver->ith_state = MACH_RCV_TOO_LARGE;
receiver->ith_msize = kmsg->ikm_header.msgh_size;
thread_go(receiver);
}
}
current_task()->messages_sent++;
return MACH_MSG_SUCCESS;
}
