void
ipc_notify_port_destroyed(
ipc_port_t port,
ipc_port_t right)
{
ipc_kmsg_t kmsg;
mach_port_destroyed_notification_t *n;
kmsg = ikm_alloc(sizeof *n);
if (kmsg == IKM_NULL) {
printf("dropped port-destroyed (%p, %p)\n",
port, right);
ipc_port_release_sonce(port);
ipc_port_release_receive(right);
return;
}
ikm_init(kmsg, sizeof *n);
n = (mach_port_destroyed_notification_t *) kmsg->ikm_header;
*n = ipc_notify_port_destroyed_template;
n->not_header.msgh_remote_port = (mach_port_t) port;
n->not_port.name = (mach_port_t)right;
ipc_mqueue_send_always(kmsg);
}
void
ipc_notify_send_once(ipc_port_t port)
{
ipc_kmsg_t kmsg;
mach_send_once_notification_t *n;
kmsg = ikm_alloc(sizeof *n);
if (kmsg == IKM_NULL) {
printf("dropped send-once (0x%p)\n", port);
ipc_port_release_sonce(port);
return;
}
ikm_init(kmsg, sizeof *n);
n = (mach_send_once_notification_t *) &kmsg->ikm_header;
*n = ipc_notify_send_once_template;
n->not_header.msgh_remote_port = (mach_port_t) port;
ipc_mqueue_send_always(kmsg);
}
void
ipc_notify_dead_name(
ipc_port_t port,
mach_port_name_t name)
{
ipc_kmsg_t kmsg;
mach_dead_name_notification_t *n;
kmsg = ikm_alloc(sizeof *n);
if (kmsg == IKM_NULL) {
printf("dropped dead-name (%p, 0x%x)\n", port, name);
ipc_port_release_sonce(port);
return;
}
ikm_init(kmsg, sizeof *n);
n = (mach_dead_name_notification_t *) kmsg->ikm_header;
*n = ipc_notify_dead_name_template;
n->not_header.msgh_remote_port = (mach_port_t) port;
n->not_port = name;
ipc_mqueue_send_always(kmsg);
}
void
ipc_notify_no_senders(
ipc_port_t port,
mach_port_mscount_t mscount)
{
ipc_kmsg_t kmsg;
mach_no_senders_notification_t *n;
kmsg = ikm_alloc(sizeof *n);
if (kmsg == IKM_NULL) {
printf("dropped no-senders (%p, %u)\n", port, mscount);
ipc_port_release_sonce(port);
return;
}
ikm_init(kmsg, sizeof *n);
n = (mach_no_senders_notification_t *) kmsg->ikm_header;
*n = ipc_notify_no_senders_template;
n->not_header.msgh_remote_port = (mach_port_t) port;
n->not_count = mscount;
ipc_mqueue_send_always(kmsg);
}
ipc_kmsg_t
ipc_kobject_server(
ipc_kmsg_t request)
{
mach_msg_size_t reply_size;
ipc_kmsg_t reply;
kern_return_t kr;
mig_routine_t routine;
ipc_port_t *destp;
mach_msg_format_0_trailer_t *trailer;
register mig_hash_t *ptr;
#if MACH_RT
boolean_t reply_rt;
#endif /* MACH_RT */
unsigned int th;
/* Only fetch current thread if ETAP is configured */
ETAP_DATA_LOAD(th, current_thread());
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_BEGIN,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
/*
* Find out corresponding mig_hash entry if any
*/
{
register int key = request->ikm_header.msgh_id;
register int i = MIG_HASH(key);
register int max_iter = mig_table_max_displ;
do
ptr = &mig_buckets[i++ % MAX_MIG_ENTRIES];
while (key != ptr->num && ptr->num && --max_iter);
if (!ptr->routine || key != ptr->num) {
ptr = (mig_hash_t *)0;
reply_size = mig_reply_size;
} else {
reply_size = ptr->size;
#if MACH_COUNTER
ptr->callcount++;
#endif
}
}
/* round up for ikm_cache; overhead is added by ikm_alloc */
if ((reply_size += MAX_TRAILER_SIZE) < IKM_SAVED_MSG_SIZE)
reply_size = IKM_SAVED_MSG_SIZE;
#if MACH_RT
reply_rt =
IP_VALID((ipc_port_t)request->ikm_header.msgh_local_port) ?
IP_RT((ipc_port_t)request->ikm_header.msgh_local_port) :
FALSE;
if (reply_rt)
reply = ikm_rtalloc(reply_size);
else
#endif /* MACH_RT */
reply = ikm_alloc(reply_size);
if (reply == IKM_NULL) {
printf("ipc_kobject_server: dropping request\n");
ipc_kmsg_destroy(request);
return IKM_NULL;
}
ikm_init(reply, reply_size);
#if DIPC
reply->ikm_handle = HANDLE_NULL;
#endif /* DIPC */
/*
* Initialize reply message.
*/
{
#define InP ((mach_msg_header_t *) &request->ikm_header)
#define OutP ((mig_reply_error_t *) &reply->ikm_header)
OutP->NDR = NDR_record;
OutP->Head.msgh_size = sizeof(mig_reply_error_t);
OutP->Head.msgh_bits =
MACH_MSGH_BITS(MACH_MSGH_BITS_LOCAL(InP->msgh_bits), 0);
OutP->Head.msgh_remote_port = InP->msgh_local_port;
OutP->Head.msgh_local_port = MACH_PORT_NULL;
OutP->Head.msgh_id = InP->msgh_id + 100;
#if MACH_RT
if (reply_rt)
KMSG_MARK_RT(reply);
#endif /* MACH_RT */
#undef InP
#undef OutP
}
/*
* Find the routine to call, and call it
* to perform the kernel function
*/
{
if (ptr)
(*ptr->routine)(&request->ikm_header, &reply->ikm_header);
else {
if (!ipc_kobject_notify(&request->ikm_header, &reply->ikm_header)){
#if MACH_IPC_TEST
printf("ipc_kobject_server: bogus kernel message, id=%d\n",
request->ikm_header.msgh_id);
#endif /* MACH_IPC_TEST */
((mig_reply_error_t *) &reply->ikm_header)->RetCode
= MIG_BAD_ID;
}
}
}
/*
* Destroy destination. The following code differs from
* ipc_object_destroy in that we release the send-once
* right instead of generating a send-once notification
* (which would bring us here again, creating a loop).
* It also differs in that we only expect send or
* send-once rights, never receive rights.
*
* We set msgh_remote_port to IP_NULL so that the kmsg
* destroy routines don't try to destroy the port twice.
*/
destp = (ipc_port_t *) &request->ikm_header.msgh_remote_port;
switch (MACH_MSGH_BITS_REMOTE(request->ikm_header.msgh_bits)) {
case MACH_MSG_TYPE_PORT_SEND:
ipc_port_release_send(*destp);
break;
case MACH_MSG_TYPE_PORT_SEND_ONCE:
ipc_port_release_sonce(*destp);
break;
default:
panic("ipc_object_destroy: strange destination rights");
}
*destp = IP_NULL;
if (!(reply->ikm_header.msgh_bits & MACH_MSGH_BITS_COMPLEX) &&
((mig_reply_error_t *) &reply->ikm_header)->RetCode != KERN_SUCCESS)
kr = ((mig_reply_error_t *) &reply->ikm_header)->RetCode;
else
kr = KERN_SUCCESS;
if ((kr == KERN_SUCCESS) || (kr == MIG_NO_REPLY)) {
/*
* The server function is responsible for the contents
* of the message. The reply port right is moved
* to the reply message, and we have deallocated
* the destination port right, so we just need
* to free the kmsg.
*/
/*
* Like ipc_kmsg_put, but without the copyout. Also,
* messages to the kernel will never have been allocated
* from the rt_zone.
*/
ikm_check_initialized(request, request->ikm_size);
if (request->ikm_size != IKM_SAVED_KMSG_SIZE ||
!ikm_cache_put (request)) {
ikm_free(request);
}
} else {
/*
* The message contents of the request are intact.
* Destroy everthing except the reply port right,
* which is needed in the reply message.
*/
request->ikm_header.msgh_local_port = MACH_PORT_NULL;
ipc_kmsg_destroy(request);
}
if (kr == MIG_NO_REPLY) {
/*
* The server function will send a reply message
* using the reply port right, which it has saved.
*/
ikm_free(reply);
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_END,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
return IKM_NULL;
} else if (!IP_VALID((ipc_port_t)reply->ikm_header.msgh_remote_port)) {
/*
* Can't queue the reply message if the destination
* (the reply port) isn't valid.
*/
ipc_kmsg_destroy(reply);
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_END,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
return IKM_NULL;
}
trailer = (mach_msg_format_0_trailer_t *)
((vm_offset_t)&reply->ikm_header + (int)reply->ikm_header.msgh_size);
trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
ETAP_PROBE_DATA(ETAP_P_SYSCALL_MACH,
EVENT_END,
((thread_t) th),
&request->ikm_header.msgh_id,
sizeof(int));
return reply;
}
void
exception_raise(
ipc_port_t dest_port,
ipc_port_t thread_port,
ipc_port_t task_port,
integer_t _exception,
integer_t code,
integer_t subcode)
{
ipc_thread_t self = current_thread();
ipc_thread_t receiver;
ipc_port_t reply_port;
ipc_mqueue_t dest_mqueue;
ipc_mqueue_t reply_mqueue;
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
assert(IP_VALID(dest_port));
/*
* We will eventually need a message buffer.
* Grab the buffer now, while nothing is locked.
* This buffer will get handed to the exception server,
* and it will give the buffer back with its reply.
*/
kmsg = ikm_cache();
if (kmsg != IKM_NULL) {
ikm_cache() = IKM_NULL;
ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
} else {
kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
if (kmsg == IKM_NULL)
panic("exception_raise");
ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
}
/*
* We need a reply port for the RPC.
* Check first for a cached port.
*/
ith_lock(self);
assert(self->ith_self != IP_NULL);
reply_port = self->ith_rpc_reply;
if (reply_port == IP_NULL) {
ith_unlock(self);
reply_port = ipc_port_alloc_reply();
ith_lock(self);
if ((reply_port == IP_NULL) ||
(self->ith_rpc_reply != IP_NULL))
panic("exception_raise");
self->ith_rpc_reply = reply_port;
}
ip_lock(reply_port);
assert(ip_active(reply_port));
ith_unlock(self);
/*
* Make a naked send-once right for the reply port,
* to hand to the exception server.
* Make an extra reference for the reply port,
* to receive on. This protects us against
* mach_msg_abort_rpc.
*/
reply_port->ip_sorights++;
ip_reference(reply_port);
ip_reference(reply_port);
self->ith_port = reply_port;
reply_mqueue = &reply_port->ip_messages;
imq_lock(reply_mqueue);
assert(ipc_kmsg_queue_empty(&reply_mqueue->imq_messages));
ip_unlock(reply_port);
/*
* Make sure we can queue to the destination port.
*/
if (!ip_lock_try(dest_port)) {
imq_unlock(reply_mqueue);
goto slow_exception_raise;
}
if (!ip_active(dest_port) ||
(dest_port->ip_receiver == ipc_space_kernel)) {
imq_unlock(reply_mqueue);
ip_unlock(dest_port);
goto slow_exception_raise;
}
/*
* Find the destination message queue.
*/
{
ipc_pset_t dest_pset;
dest_pset = dest_port->ip_pset;
if (dest_pset == IPS_NULL)
dest_mqueue = &dest_port->ip_messages;
else
dest_mqueue = &dest_pset->ips_messages;
}
if (!imq_lock_try(dest_mqueue)) {
imq_unlock(reply_mqueue);
ip_unlock(dest_port);
goto slow_exception_raise;
}
/*
* Safe to unlock dest_port, because we hold
* dest_mqueue locked. We never bother changing
* dest_port->ip_msgcount.
*/
ip_unlock(dest_port);
receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
if ((receiver == ITH_NULL) ||
!((receiver->swap_func == (void (*)()) mach_msg_continue) ||
((receiver->swap_func ==
(void (*)()) mach_msg_receive_continue) &&
(sizeof(struct mach_exception) <= receiver->ith_msize) &&
((receiver->ith_option & MACH_RCV_NOTIFY) == 0))) ||
!thread_handoff(self, exception_raise_continue, receiver)) {
imq_unlock(reply_mqueue);
imq_unlock(dest_mqueue);
goto slow_exception_raise;
}
counter(c_exception_raise_block++);
assert(current_thread() == receiver);
/*
* We need to finish preparing self for its
* time asleep in reply_mqueue. self is left
* holding the extra ref for reply_port.
*/
ipc_thread_enqueue_macro(&reply_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(reply_mqueue);
/*
* Finish extracting receiver from dest_mqueue.
*/
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
imq_unlock(dest_mqueue);
/*
* Release the receiver's reference for his object.
*/
{
ipc_object_t object = receiver->ith_object;
io_lock(object);
io_release(object);
io_check_unlock(object);
}
{
struct mach_exception *exc =
(struct mach_exception *) &kmsg->ikm_header;
ipc_space_t space = receiver->task->itk_space;
/*
* We are running as the receiver now. We hold
* the following resources, which must be consumed:
* kmsg, send-once right for reply_port
* send rights for dest_port, thread_port, task_port
* Synthesize a kmsg for copyout to the receiver.
*/
exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PORT_SEND) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_size = sizeof *exc;
/* exc->Head.msgh_remote_port later */
/* exc->Head.msgh_local_port later */
exc->Head.msgh_seqno = 0;
exc->Head.msgh_id = MACH_EXCEPTION_ID;
exc->threadType = exc_port_proto;
/* exc->thread later */
exc->taskType = exc_port_proto;
/* exc->task later */
exc->exceptionType = exc_code_proto;
exc->exception = _exception;
exc->codeType = exc_code_proto;
exc->code = code;
exc->subcodeType = exc_code_proto;
exc->subcode = subcode;
/*
* Check that the receiver can handle the message.
*/
if (receiver->ith_rcv_size < sizeof(struct mach_exception)) {
/*
* ipc_kmsg_destroy is a handy way to consume
* the resources we hold, but it requires setup.
*/
exc->Head.msgh_bits =
(MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
exc->thread = (mach_port_t) thread_port;
exc->task = (mach_port_t) task_port;
ipc_kmsg_destroy(kmsg);
thread_syscall_return(MACH_RCV_TOO_LARGE);
/*NOTREACHED*/
}
is_write_lock(space);
assert(space->is_active);
/*
* To do an atomic copyout, need simultaneous
* locks on both ports and the space.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
abort_copyout:
ip_unlock(dest_port);
is_write_unlock(space);
/*
* Oh well, we have to do the header the slow way.
* First make it look like it's in-transit.
*/
exc->Head.msgh_bits =
(MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
mr = ipc_kmsg_copyout_header(&exc->Head, space,
MACH_PORT_NULL);
if (mr == MACH_MSG_SUCCESS)
goto copyout_body;
/*
* Ack! Prepare for ipc_kmsg_copyout_dest.
* It will consume thread_port and task_port.
*/
exc->thread = (mach_port_t) thread_port;
exc->task = (mach_port_t) task_port;
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(receiver->ith_msg, kmsg,
sizeof(mach_msg_header_t));
thread_syscall_return(mr);
/*NOTREACHED*/
}
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
goto abort_copyout;
}
assert(reply_port->ip_sorights > 0);
ip_unlock(reply_port);
{
kern_return_t kr;
ipc_entry_t entry;
kr = ipc_entry_get (space, &exc->Head.msgh_remote_port, &entry);
if (kr)
goto abort_copyout;
{
mach_port_gen_t gen;
assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
gen = entry->ie_bits + IE_BITS_GEN_ONE;
/* optimized ipc_right_copyout */
entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
}
entry->ie_object = (ipc_object_t) reply_port;
is_write_unlock(space);
}
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_srights > 0);
ip_release(dest_port);
exc->Head.msgh_local_port =
((dest_port->ip_receiver == space) ?
dest_port->ip_receiver_name : MACH_PORT_NULL);
if ((--dest_port->ip_srights == 0) &&
(dest_port->ip_nsrequest != IP_NULL)) {
ipc_port_t nsrequest;
mach_port_mscount_t mscount;
/* a rather rare case */
nsrequest = dest_port->ip_nsrequest;
mscount = dest_port->ip_mscount;
dest_port->ip_nsrequest = IP_NULL;
ip_unlock(dest_port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(dest_port);
copyout_body:
/*
* Optimized version of ipc_kmsg_copyout_body,
* to handle the two ports in the body.
*/
mr = (ipc_kmsg_copyout_object(space, (ipc_object_t) thread_port,
MACH_MSG_TYPE_PORT_SEND, &exc->thread) |
ipc_kmsg_copyout_object(space, (ipc_object_t) task_port,
MACH_MSG_TYPE_PORT_SEND, &exc->task));
if (mr != MACH_MSG_SUCCESS) {
(void) ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr | MACH_RCV_BODY_ERROR);
/*NOTREACHED*/
}
}
/*
* Optimized version of ipc_kmsg_put.
* We must check ikm_cache after copyoutmsg.
*/
ikm_check_initialized(kmsg, kmsg->ikm_size);
assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
if (copyoutmsg(&kmsg->ikm_header, receiver->ith_msg,
sizeof(struct mach_exception)) ||
(ikm_cache() != IKM_NULL)) {
mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
ikm_cache() = kmsg;
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
#ifndef __GNUC__
return; /* help for the compiler */
#endif
slow_exception_raise: {
struct mach_exception *exc =
(struct mach_exception *) &kmsg->ikm_header;
ipc_kmsg_t reply_kmsg;
mach_port_seqno_t reply_seqno;
exception_raise_misses++;
/*
* We hold the following resources, which must be consumed:
* kmsg, send-once right and ref for reply_port
* send rights for dest_port, thread_port, task_port
* Synthesize a kmsg to send.
*/
exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_size = sizeof *exc;
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
exc->Head.msgh_seqno = 0;
exc->Head.msgh_id = MACH_EXCEPTION_ID;
exc->threadType = exc_port_proto;
exc->thread = (mach_port_t) thread_port;
exc->taskType = exc_port_proto;
exc->task = (mach_port_t) task_port;
exc->exceptionType = exc_code_proto;
exc->exception = _exception;
exc->codeType = exc_code_proto;
exc->code = code;
exc->subcodeType = exc_code_proto;
exc->subcode = subcode;
ipc_mqueue_send_always(kmsg);
/*
* We are left with a ref for reply_port,
* which we use to receive the reply message.
*/
ip_lock(reply_port);
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
exception_raise_continue_slow(MACH_RCV_PORT_DIED, IKM_NULL, /*dummy*/0);
/*NOTREACHED*/
}
imq_lock(reply_mqueue);
ip_unlock(reply_port);
mr = ipc_mqueue_receive(reply_mqueue, MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, exception_raise_continue,
&reply_kmsg, &reply_seqno);
/* reply_mqueue is unlocked */
exception_raise_continue_slow(mr, reply_kmsg, reply_seqno);
/*NOTREACHED*/
}
}
kern_return_t
mach_port_allocate_full(
ipc_space_t space,
mach_port_right_t right,
mach_port_t proto,
mach_port_qos_t *qosp,
mach_port_name_t *namep)
{
ipc_kmsg_t kmsg;
kern_return_t kr;
if (space == IS_NULL)
return (KERN_INVALID_TASK);
if (proto != MACH_PORT_NULL)
return (KERN_INVALID_VALUE);
if (qosp->name) {
if (!MACH_PORT_VALID (*namep))
return (KERN_INVALID_VALUE);
if (is_fast_space (space))
return (KERN_FAILURE);
}
if (qosp->prealloc) {
mach_msg_size_t size = qosp->len + MAX_TRAILER_SIZE;
if (right != MACH_PORT_RIGHT_RECEIVE)
return (KERN_INVALID_VALUE);
kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(size));
if (kmsg == IKM_NULL)
return (KERN_RESOURCE_SHORTAGE);
ikm_init(kmsg, size);
}
switch (right) {
case MACH_PORT_RIGHT_RECEIVE:
{
ipc_port_t port;
if (qosp->name)
kr = ipc_port_alloc_name(space, *namep, &port);
else
kr = ipc_port_alloc(space, namep, &port);
if (kr == KERN_SUCCESS) {
if (qosp->prealloc)
ipc_kmsg_set_prealloc(kmsg, port);
ip_unlock(port);
} else if (qosp->prealloc)
ipc_kmsg_free(kmsg);
break;
}
case MACH_PORT_RIGHT_PORT_SET:
{
ipc_pset_t pset;
if (qosp->name)
kr = ipc_pset_alloc_name(space, *namep, &pset);
else
kr = ipc_pset_alloc(space, namep, &pset);
if (kr == KERN_SUCCESS)
ips_unlock(pset);
break;
}
case MACH_PORT_RIGHT_DEAD_NAME:
kr = ipc_object_alloc_dead(space, namep);
break;
default:
kr = KERN_INVALID_VALUE;
break;
}
return (kr);
}
