ipc_port_t
ipc_port_copy_send(
ipc_port_t port)
{
ipc_port_t sright;
if (!IP_VALID(port))
return port;
ip_lock(port);
if (ip_active(port)) {
assert(port->ip_srights > 0);
ip_reference(port);
port->ip_srights++;
sright = port;
} else
sright = IP_DEAD;
ip_unlock(port);
return sright;
}
/*
* Routine: convert_port_to_task_name
* Purpose:
* Convert from a port to a task name.
* Doesn't consume the port ref; produces a task name ref,
* which may be null.
* Conditions:
* Nothing locked.
*/
task_name_t
convert_port_to_task_name(
ipc_port_t port)
{
task_name_t task = TASK_NULL;
if (IP_VALID(port)) {
ip_lock(port);
if ( ip_active(port) &&
(ip_kotype(port) == IKOT_TASK ||
ip_kotype(port) == IKOT_TASK_NAME)) {
task = (task_name_t)port->ip_kobject;
assert(task != TASK_NAME_NULL);
task_reference_internal(task);
}
ip_unlock(port);
}
return (task);
}
/*
* Called with memory_object locked. Unlocks memory_object.
*/
ipc_port_t
xmm_object_by_memory_object_remote(
ipc_port_t memory_object)
{
kern_return_t kr;
ipc_port_t xmm_object;
#if DIPC
node_name node;
#endif /* DIPC */
assert(IP_WAS_REMOTE(memory_object));
#if DIPC
node = DIPC_ORIGIN_NODE(memory_object);
ip_unlock(memory_object);
kr = r_xmm_remote_object_by_memory_object(dipc_host_priv_port(node),
memory_object,
&xmm_object);
#else /* DIPC */
panic("xmm_object_by_memory_object_remote: no transport");
#endif /* DIPC */
assert(kr == KERN_SUCCESS);
return xmm_object;
}
kern_return_t
mach_port_get_srights(
ipc_space_t space,
mach_port_t name,
mach_port_rights_t *srightsp)
{
ipc_port_t port;
kern_return_t kr;
mach_port_rights_t srights;
if (space == IS_NULL)
return KERN_INVALID_TASK;
kr = ipc_port_translate_receive(space, name, &port);
if (kr != KERN_SUCCESS)
return kr;
/* port is locked and active */
srights = port->ip_srights;
ip_unlock(port);
*srightsp = srights;
return KERN_SUCCESS;
}
void
ipc_object_copyin_from_kernel(
ipc_object_t object,
mach_msg_type_name_t msgt_name)
{
assert(IO_VALID(object));
switch (msgt_name) {
case MACH_MSG_TYPE_MOVE_RECEIVE: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name != MACH_PORT_NULL);
assert(port->ip_receiver == ipc_space_kernel);
/* relevant part of ipc_port_clear_receiver */
ipc_port_set_mscount(port, 0);
port->ip_receiver_name = MACH_PORT_NULL;
port->ip_destination = IP_NULL;
ipc_port_flag_protected_payload_clear(port);
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_COPY_SEND: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
if (ip_active(port)) {
assert(port->ip_srights > 0);
port->ip_srights++;
}
ip_reference(port);
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_MAKE_SEND: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name != MACH_PORT_NULL);
assert(port->ip_receiver == ipc_space_kernel);
ip_reference(port);
port->ip_mscount++;
port->ip_srights++;
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_MOVE_SEND:
/* move naked send right into the message */
break;
case MACH_MSG_TYPE_MAKE_SEND_ONCE: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name != MACH_PORT_NULL);
assert(port->ip_receiver == ipc_space_kernel);
ip_reference(port);
port->ip_sorights++;
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_MOVE_SEND_ONCE:
/* move naked send-once right into the message */
break;
default:
#if MACH_ASSERT
assert(!"ipc_object_copyin_from_kernel: strange rights");
#else
panic("ipc_object_copyin_from_kernel: strange rights");
#endif
}
}
/*
* XXX who cares about send rights anymore XXX
* Return send right for xmm object corresponding to memory object.
* This is to be consumed when using xmm object to set up init,
* either via move_send dest in proxy_init, or explicit deallocation
* in local case.
* Also returns one xmm_object ref, to be given to svm layer and
* released there upon termination via xmm_object_release.
* Also returns one xmm obj ref, to be consumed by xmm_obj_allocate
* in either _proxy_init or xmm_memory_object_init.
*
* Create xmm object if necessary.
* Memory object holds a send right to xmm object as well, which is released
* when xmm object refs drop to 0. No-senders then triggers XXXXXX
* svm deallocation.
*/
ipc_port_t
xmm_object_by_memory_object(
ipc_port_t memory_object)
{
ipc_port_t xmm_object, old_xmm_object;
xmm_obj_t mobj;
/*
* We always create the svm stack at the current location of the
* memory object. We may have to chase it down if it's migrating.
*
* The memory_object principal node is the one true source
* of knowledge about whether an svm stack exists.
*/
ip_lock(memory_object);
xmm_object_wait(memory_object);
if (IP_WAS_REMOTE(memory_object)) {
/* the following call inherits the lock */
return xmm_object_by_memory_object_remote(memory_object);
}
/*
* If there is already an xmm_object associated with this
* memory_object, return it, after taking a send-right reference
* which will be given (moved, if necessary) to the caller.
*/
xmm_object = xmm_object_copy(memory_object);
if (xmm_object != IP_NULL) {
ip_unlock(memory_object);
assert(!IP_IS_REMOTE(xmm_object));
return xmm_object;
}
/*
* Check kobject type, to foil attempts to map in inappropriate
* kernel objects (like task ports).
*/
if (ip_kotype(memory_object) != IKOT_NONE &&
ip_kotype(memory_object) != IKOT_PAGER) {
ip_unlock(memory_object);
return IP_NULL;
}
/*
* No xmm object is currently associcated with memory object.
* Unlock memory object port, and create an xmm obj stack.
* and a corresponding xmm obj stack.
*
* XXX
* Should deallocate things if this call fails part-way.
*/
ip_unlock(memory_object);
xmm_user_create(memory_object, &mobj);
xmm_svm_create(mobj, memory_object, &mobj);
/*
* Create an xmm object and associate it with stack.
* It will have one send right.
*/
xmm_object = xmm_object_allocate(mobj);
if (xmm_object == IP_NULL) {
panic("xmm_mo_create: xmm_object_allocate");
return IP_NULL;
}
ip_lock(memory_object);
assert(!IP_WAS_REMOTE(memory_object));
/*
* If we lost the race to create the stack, discard ours
* and use the one already created. Otherwise, associate
* our xmm object and stack with the memory object,
* by giving the memory object the send right to the xmm object.
*/
old_xmm_object = xmm_object_copy(memory_object);
if (old_xmm_object != IP_NULL) {
ip_unlock(memory_object);
xmm_object_destroy(xmm_object, mobj);
xmm_object = old_xmm_object;
} else {
xmm_object_set(memory_object, xmm_object, TRUE);
ip_unlock(memory_object);
}
/*
* Return the xmm object send right.
*/
return xmm_object;
}
boolean_t
ipc_right_copyin_check(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry,
mach_msg_type_name_t msgt_name)
{
ipc_entry_bits_t bits = entry->ie_bits;
assert(space->is_active);
switch (msgt_name) {
case MACH_MSG_TYPE_MAKE_SEND:
case MACH_MSG_TYPE_MAKE_SEND_ONCE:
case MACH_MSG_TYPE_MOVE_RECEIVE:
if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
return FALSE;
break;
case MACH_MSG_TYPE_COPY_SEND:
case MACH_MSG_TYPE_MOVE_SEND:
case MACH_MSG_TYPE_MOVE_SEND_ONCE: {
ipc_port_t port;
boolean_t active;
if (bits & MACH_PORT_TYPE_DEAD_NAME)
break;
if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0)
return FALSE;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
ip_lock(port);
active = ip_active(port);
ip_unlock(port);
if (!active) {
break;
}
if (msgt_name == MACH_MSG_TYPE_MOVE_SEND_ONCE) {
if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0)
return FALSE;
} else {
if ((bits & MACH_PORT_TYPE_SEND) == 0)
return FALSE;
}
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_copyin_check: strange rights");
#else
panic("ipc_right_copyin_check: strange rights");
#endif
}
return TRUE;
}
void
ipc_port_destroy(
ipc_port_t port)
{
ipc_port_t pdrequest, nsrequest;
ipc_mqueue_t mqueue;
ipc_kmsg_t kmsg;
#if IMPORTANCE_INHERITANCE
task_t release_imp_task = TASK_NULL;
thread_t self = current_thread();
boolean_t top = (self->ith_assertions == 0);
natural_t assertcnt = 0;
#endif /* IMPORTANCE_INHERITANCE */
assert(ip_active(port));
/* port->ip_receiver_name is garbage */
/* port->ip_receiver/port->ip_destination is garbage */
assert(port->ip_pset_count == 0);
assert(port->ip_mscount == 0);
/* check for a backup port */
pdrequest = port->ip_pdrequest;
#if IMPORTANCE_INHERITANCE
/* determine how may assertions to drop and from whom */
if (port->ip_tempowner != 0) {
assert(top);
if (port->ip_taskptr != 0) {
release_imp_task = port->ip_imp_task;
port->ip_imp_task = TASK_NULL;
port->ip_taskptr = 0;
assertcnt = port->ip_impcount;
}
/* Otherwise, nothing to drop */
} else {
assert(port->ip_taskptr == 0);
assertcnt = port->ip_impcount;
if (pdrequest != IP_NULL)
/* mark in limbo for the journey */
port->ip_tempowner = 1;
}
if (top)
self->ith_assertions = assertcnt;
#endif /* IMPORTANCE_INHERITANCE */
if (pdrequest != IP_NULL) {
/* we assume the ref for pdrequest */
port->ip_pdrequest = IP_NULL;
/* make port be in limbo */
port->ip_receiver_name = MACH_PORT_NULL;
port->ip_destination = IP_NULL;
ip_unlock(port);
/* consumes our refs for port and pdrequest */
ipc_notify_port_destroyed(pdrequest, port);
goto drop_assertions;
}
/* once port is dead, we don't need to keep it locked */
port->ip_object.io_bits &= ~IO_BITS_ACTIVE;
port->ip_timestamp = ipc_port_timestamp();
/*
* If the port has a preallocated message buffer and that buffer
* is not inuse, free it. If it has an inuse one, then the kmsg
* free will detect that we freed the association and it can free it
* like a normal buffer.
*/
if (IP_PREALLOC(port)) {
ipc_port_t inuse_port;
kmsg = port->ip_premsg;
assert(kmsg != IKM_NULL);
inuse_port = ikm_prealloc_inuse_port(kmsg);
IP_CLEAR_PREALLOC(port, kmsg);
ip_unlock(port);
if (inuse_port != IP_NULL) {
assert(inuse_port == port);
} else {
ipc_kmsg_free(kmsg);
}
} else {
ip_unlock(port);
}
/* throw away no-senders request */
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL)
ipc_notify_send_once(nsrequest); /* consumes ref */
/* destroy any queued messages */
mqueue = &port->ip_messages;
ipc_mqueue_destroy(mqueue);
/* generate dead-name notifications */
ipc_port_dnnotify(port);
ipc_kobject_destroy(port);
ip_release(port); /* consume caller's ref */
drop_assertions:
#if IMPORTANCE_INHERITANCE
if (release_imp_task != TASK_NULL) {
if (assertcnt > 0) {
assert(top);
self->ith_assertions = 0;
assert(release_imp_task->imp_receiver != 0);
task_importance_drop_internal_assertion(release_imp_task, assertcnt);
}
task_deallocate(release_imp_task);
} else if (assertcnt > 0) {
if (top) {
self->ith_assertions = 0;
release_imp_task = current_task();
if (release_imp_task->imp_receiver != 0) {
task_importance_drop_internal_assertion(release_imp_task, assertcnt);
}
} else {
/* the port chain we are enqueued on should cover our assertions */
assert(assertcnt <= self->ith_assertions);
}
}
#endif /* IMPORTANCE_INHERITANCE */
}
kern_return_t
ipc_port_request_grow(
ipc_port_t port,
ipc_table_elems_t target_size)
{
ipc_table_size_t its;
ipc_port_request_t otable, ntable;
assert(ip_active(port));
otable = port->ip_requests;
if (otable == IPR_NULL)
its = &ipc_table_requests[0];
else
its = otable->ipr_size + 1;
if (target_size != ITS_SIZE_NONE) {
if ((otable != IPR_NULL) &&
(target_size <= otable->ipr_size->its_size)) {
ip_unlock(port);
return KERN_SUCCESS;
}
while ((its->its_size) && (its->its_size < target_size)) {
its++;
}
if (its->its_size == 0) {
ip_unlock(port);
return KERN_NO_SPACE;
}
}
ip_reference(port);
ip_unlock(port);
if ((its->its_size == 0) ||
((ntable = it_requests_alloc(its)) == IPR_NULL)) {
ip_release(port);
return KERN_RESOURCE_SHORTAGE;
}
ip_lock(port);
/*
* Check that port is still active and that nobody else
* has slipped in and grown the table on us. Note that
* just checking if the current table pointer == otable
* isn't sufficient; must check ipr_size.
*/
if (ip_active(port) && (port->ip_requests == otable) &&
((otable == IPR_NULL) || (otable->ipr_size+1 == its))) {
ipc_table_size_t oits;
ipc_table_elems_t osize, nsize;
ipc_port_request_index_t free, i;
/* copy old table to new table */
if (otable != IPR_NULL) {
oits = otable->ipr_size;
osize = oits->its_size;
free = otable->ipr_next;
(void) memcpy((void *)(ntable + 1),
(const void *)(otable + 1),
(osize - 1) * sizeof(struct ipc_port_request));
} else {
osize = 1;
oits = 0;
free = 0;
}
nsize = its->its_size;
assert(nsize > osize);
/* add new elements to the new table's free list */
for (i = osize; i < nsize; i++) {
ipc_port_request_t ipr = &ntable[i];
ipr->ipr_name = MACH_PORT_NULL;
ipr->ipr_next = free;
free = i;
}
ntable->ipr_next = free;
ntable->ipr_size = its;
port->ip_requests = ntable;
ip_unlock(port);
ip_release(port);
if (otable != IPR_NULL) {
it_requests_free(oits, otable);
}
} else {
ip_unlock(port);
ip_release(port);
it_requests_free(its, ntable);
}
return KERN_SUCCESS;
}
kern_return_t
ipc_right_dnrequest(
ipc_space_t space,
mach_port_t name,
boolean_t immediate,
ipc_port_t notify,
ipc_port_t *previousp)
{
ipc_port_t previous;
for (;;) {
ipc_entry_t entry;
ipc_entry_bits_t bits;
kern_return_t kr;
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
bits = entry->ie_bits;
if (bits & MACH_PORT_TYPE_PORT_RIGHTS) {
ipc_port_t port;
ipc_port_request_index_t request;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (!ipc_right_check(space, port, name, entry)) {
/* port is locked and active */
if (notify == IP_NULL) {
previous = ipc_right_dncancel_macro(
space, port, name, entry);
ip_unlock(port);
is_write_unlock(space);
break;
}
/*
* If a registered soright exists,
* want to atomically switch with it.
* If ipc_port_dncancel finds us a
* soright, then the following
* ipc_port_dnrequest will reuse
* that slot, so we are guaranteed
* not to unlock and retry.
*/
previous = ipc_right_dncancel_macro(space,
port, name, entry);
kr = ipc_port_dnrequest(port, name, notify,
&request);
if (kr != KERN_SUCCESS) {
assert(previous == IP_NULL);
is_write_unlock(space);
kr = ipc_port_dngrow(port);
/* port is unlocked */
if (kr != KERN_SUCCESS)
return kr;
continue;
}
assert(request != 0);
ip_unlock(port);
entry->ie_request = request;
is_write_unlock(space);
break;
}
bits = entry->ie_bits;
assert(bits & MACH_PORT_TYPE_DEAD_NAME);
}
if ((bits & MACH_PORT_TYPE_DEAD_NAME) &&
immediate && (notify != IP_NULL)) {
mach_port_urefs_t urefs = IE_BITS_UREFS(bits);
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
assert(urefs > 0);
if (MACH_PORT_UREFS_OVERFLOW(urefs, 1)) {
is_write_unlock(space);
return KERN_UREFS_OVERFLOW;
}
entry->ie_bits = bits + 1; /* increment urefs */
is_write_unlock(space);
ipc_notify_dead_name(notify, name);
previous = IP_NULL;
break;
}
is_write_unlock(space);
if (bits & MACH_PORT_TYPE_PORT_OR_DEAD)
return KERN_INVALID_ARGUMENT;
else
return KERN_INVALID_RIGHT;
}
*previousp = previous;
return KERN_SUCCESS;
}
kern_return_t
ipc_right_copyin(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry,
mach_msg_type_name_t msgt_name,
boolean_t deadok,
ipc_object_t *objectp,
ipc_port_t *sorightp)
{
ipc_entry_bits_t bits = entry->ie_bits;
assert(space->is_active);
switch (msgt_name) {
case MACH_MSG_TYPE_MAKE_SEND: {
ipc_port_t port;
if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
goto invalid_right;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
port->ip_mscount++;
port->ip_srights++;
ip_reference(port);
ip_unlock(port);
*objectp = (ipc_object_t) port;
*sorightp = IP_NULL;
break;
}
case MACH_MSG_TYPE_MAKE_SEND_ONCE: {
ipc_port_t port;
if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
goto invalid_right;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
port->ip_sorights++;
ip_reference(port);
ip_unlock(port);
*objectp = (ipc_object_t) port;
*sorightp = IP_NULL;
break;
}
case MACH_MSG_TYPE_MOVE_RECEIVE: {
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
goto invalid_right;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
if (bits & MACH_PORT_TYPE_SEND) {
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
assert(IE_BITS_UREFS(bits) > 0);
assert(port->ip_srights > 0);
entry->ie_name = name;
ipc_reverse_insert(space, (ipc_object_t) port, entry);
ip_reference(port);
} else {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE);
assert(IE_BITS_UREFS(bits) == 0);
dnrequest = ipc_right_dncancel_macro(space, port,
name, entry);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
entry->ie_object = IO_NULL;
}
entry->ie_bits = bits &~ MACH_PORT_TYPE_RECEIVE;
ipc_port_clear_receiver(port);
port->ip_receiver_name = MACH_PORT_NULL;
port->ip_destination = IP_NULL;
/*
* Clear the protected payload field to retain
* the behavior of mach_msg.
*/
ipc_port_flag_protected_payload_clear(port);
ip_unlock(port);
*objectp = (ipc_object_t) port;
*sorightp = dnrequest;
break;
}
case MACH_MSG_TYPE_COPY_SEND: {
ipc_port_t port;
if (bits & MACH_PORT_TYPE_DEAD_NAME)
goto copy_dead;
/* allow for dead send-once rights */
if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0)
goto invalid_right;
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
goto copy_dead;
}
/* port is locked and active */
if ((bits & MACH_PORT_TYPE_SEND) == 0) {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
assert(port->ip_sorights > 0);
ip_unlock(port);
goto invalid_right;
}
assert(port->ip_srights > 0);
port->ip_srights++;
ip_reference(port);
ip_unlock(port);
*objectp = (ipc_object_t) port;
*sorightp = IP_NULL;
break;
}
case MACH_MSG_TYPE_MOVE_SEND: {
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
if (bits & MACH_PORT_TYPE_DEAD_NAME)
goto move_dead;
/* allow for dead send-once rights */
if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0)
goto invalid_right;
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
goto move_dead;
}
/* port is locked and active */
if ((bits & MACH_PORT_TYPE_SEND) == 0) {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
assert(port->ip_sorights > 0);
ip_unlock(port);
goto invalid_right;
}
assert(port->ip_srights > 0);
if (IE_BITS_UREFS(bits) == 1) {
if (bits & MACH_PORT_TYPE_RECEIVE) {
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
ip_reference(port);
} else {
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND);
dnrequest = ipc_right_dncancel_macro(
space, port, name, entry);
ipc_reverse_remove(space, (ipc_object_t) port);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
entry->ie_object = IO_NULL;
}
entry->ie_bits = bits &~
(IE_BITS_UREFS_MASK|MACH_PORT_TYPE_SEND);
} else {
port->ip_srights++;
ip_reference(port);
entry->ie_bits = bits-1; /* decrement urefs */
}
ip_unlock(port);
*objectp = (ipc_object_t) port;
*sorightp = dnrequest;
break;
}
case MACH_MSG_TYPE_MOVE_SEND_ONCE: {
ipc_port_t port;
ipc_port_t dnrequest;
if (bits & MACH_PORT_TYPE_DEAD_NAME)
goto move_dead;
/* allow for dead send rights */
if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0)
goto invalid_right;
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
goto move_dead;
}
/* port is locked and active */
if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0) {
assert(bits & MACH_PORT_TYPE_SEND);
assert(port->ip_srights > 0);
ip_unlock(port);
goto invalid_right;
}
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
assert(IE_BITS_UREFS(bits) == 1);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(port->ip_sorights > 0);
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
ip_unlock(port);
entry->ie_object = IO_NULL;
entry->ie_bits = bits &~ MACH_PORT_TYPE_SEND_ONCE;
*objectp = (ipc_object_t) port;
*sorightp = dnrequest;
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_copyin: strange rights");
#else
panic("ipc_right_copyin: strange rights");
#endif
}
return KERN_SUCCESS;
copy_dead:
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
assert(IE_BITS_UREFS(bits) > 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(entry->ie_request == 0);
assert(entry->ie_object == 0);
if (!deadok)
goto invalid_right;
*objectp = IO_DEAD;
*sorightp = IP_NULL;
return KERN_SUCCESS;
move_dead:
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
assert(IE_BITS_UREFS(bits) > 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(entry->ie_request == 0);
assert(entry->ie_object == 0);
if (!deadok)
goto invalid_right;
if (IE_BITS_UREFS(bits) == 1)
entry->ie_bits = bits &~ MACH_PORT_TYPE_DEAD_NAME;
else
entry->ie_bits = bits-1; /* decrement urefs */
*objectp = IO_DEAD;
*sorightp = IP_NULL;
return KERN_SUCCESS;
invalid_right:
return KERN_INVALID_RIGHT;
}
kern_return_t
ipc_right_dealloc(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry)
{
ipc_entry_bits_t bits = entry->ie_bits;
mach_port_type_t type = IE_BITS_TYPE(bits);
assert(space->is_active);
switch (type) {
case MACH_PORT_TYPE_DEAD_NAME: {
dead_name:
assert(IE_BITS_UREFS(bits) > 0);
assert(entry->ie_request == 0);
assert(entry->ie_object == IO_NULL);
assert((bits & IE_BITS_MAREQUEST) == 0);
if (IE_BITS_UREFS(bits) == 1)
ipc_entry_dealloc(space, name, entry);
else
entry->ie_bits = bits-1; /* decrement urefs */
is_write_unlock(space);
break;
}
case MACH_PORT_TYPE_SEND_ONCE: {
ipc_port_t port, dnrequest;
assert(IE_BITS_UREFS(bits) == 1);
assert((bits & IE_BITS_MAREQUEST) == 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
goto dead_name;
}
/* port is locked and active */
assert(port->ip_sorights > 0);
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
ip_unlock(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
ipc_notify_send_once(port);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_TYPE_SEND: {
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
goto dead_name;
}
/* port is locked and active */
assert(port->ip_srights > 0);
if (IE_BITS_UREFS(bits) == 1) {
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
dnrequest = ipc_right_dncancel_macro(space, port,
name, entry);
ipc_reverse_remove(space, (ipc_object_t) port);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
ip_release(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
} else
entry->ie_bits = bits-1; /* decrement urefs */
ip_unlock(port); /* even if dropped a ref, port is active */
is_write_unlock(space);
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_TYPE_SEND_RECEIVE: {
ipc_port_t port;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
assert(port->ip_srights > 0);
if (IE_BITS_UREFS(bits) == 1) {
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK|
MACH_PORT_TYPE_SEND);
} else
entry->ie_bits = bits-1; /* decrement urefs */
ip_unlock(port);
is_write_unlock(space);
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
break;
}
default:
is_write_unlock(space);
return KERN_INVALID_RIGHT;
}
return KERN_SUCCESS;
}
kern_return_t
ipc_right_delta(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry,
mach_port_right_t right,
mach_port_delta_t delta)
{
ipc_entry_bits_t bits = entry->ie_bits;
assert(space->is_active);
assert(right < MACH_PORT_RIGHT_NUMBER);
/* Rights-specific restrictions and operations. */
switch (right) {
case MACH_PORT_RIGHT_PORT_SET: {
ipc_pset_t pset;
if ((bits & MACH_PORT_TYPE_PORT_SET) == 0)
goto invalid_right;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_PORT_SET);
assert(IE_BITS_UREFS(bits) == 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(entry->ie_request == 0);
if (delta == 0)
goto success;
if (delta != -1)
goto invalid_value;
pset = (ipc_pset_t) entry->ie_object;
assert(pset != IPS_NULL);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
ips_lock(pset);
assert(ips_active(pset));
is_write_unlock(space);
ipc_pset_destroy(pset); /* consumes ref, unlocks */
break;
}
case MACH_PORT_RIGHT_RECEIVE: {
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
goto invalid_right;
if (delta == 0)
goto success;
if (delta != -1)
goto invalid_value;
if (bits & IE_BITS_MAREQUEST) {
bits &= ~IE_BITS_MAREQUEST;
ipc_marequest_cancel(space, name);
}
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
/*
* The port lock is needed for ipc_right_dncancel;
* otherwise, we wouldn't have to take the lock
* until just before dropping the space lock.
*/
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
if (bits & MACH_PORT_TYPE_SEND) {
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
assert(IE_BITS_UREFS(bits) > 0);
assert(IE_BITS_UREFS(bits) < MACH_PORT_UREFS_MAX);
assert(port->ip_srights > 0);
/*
* The remaining send right turns into a
* dead name. Notice we don't decrement
* ip_srights, generate a no-senders notif,
* or use ipc_right_dncancel, because the
* port is destroyed "first".
*/
bits &= ~IE_BITS_TYPE_MASK;
bits |= MACH_PORT_TYPE_DEAD_NAME;
if (entry->ie_request != 0) {
entry->ie_request = 0;
bits++; /* increment urefs */
}
entry->ie_bits = bits;
entry->ie_object = IO_NULL;
} else {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE);
assert(IE_BITS_UREFS(bits) == 0);
dnrequest = ipc_right_dncancel_macro(space, port,
name, entry);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
}
is_write_unlock(space);
ipc_port_clear_receiver(port);
ipc_port_destroy(port); /* consumes ref, unlocks */
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_RIGHT_SEND_ONCE: {
ipc_port_t port, dnrequest;
if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0)
goto invalid_right;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
assert(IE_BITS_UREFS(bits) == 1);
assert((bits & IE_BITS_MAREQUEST) == 0);
if ((delta > 0) || (delta < -1))
goto invalid_value;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
assert(!(entry->ie_bits & MACH_PORT_TYPE_SEND_ONCE));
goto invalid_right;
}
/* port is locked and active */
assert(port->ip_sorights > 0);
if (delta == 0) {
ip_unlock(port);
goto success;
}
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
ip_unlock(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
ipc_notify_send_once(port);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_RIGHT_DEAD_NAME: {
mach_port_urefs_t urefs;
if (bits & MACH_PORT_TYPE_SEND_RIGHTS) {
ipc_port_t port;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (!ipc_right_check(space, port, name, entry)) {
/* port is locked and active */
ip_unlock(port);
goto invalid_right;
}
bits = entry->ie_bits;
} else if ((bits & MACH_PORT_TYPE_DEAD_NAME) == 0)
goto invalid_right;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
assert(IE_BITS_UREFS(bits) > 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(entry->ie_object == IO_NULL);
assert(entry->ie_request == 0);
urefs = IE_BITS_UREFS(bits);
if (MACH_PORT_UREFS_UNDERFLOW(urefs, delta))
goto invalid_value;
if (MACH_PORT_UREFS_OVERFLOW(urefs, delta))
goto urefs_overflow;
if ((urefs + delta) == 0)
ipc_entry_dealloc(space, name, entry);
else
entry->ie_bits = bits + delta;
is_write_unlock(space);
break;
}
case MACH_PORT_RIGHT_SEND: {
mach_port_urefs_t urefs;
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
if ((bits & MACH_PORT_TYPE_SEND) == 0)
goto invalid_right;
/* maximum urefs for send is MACH_PORT_UREFS_MAX-1 */
urefs = IE_BITS_UREFS(bits);
if (MACH_PORT_UREFS_UNDERFLOW(urefs, delta))
goto invalid_value;
if (MACH_PORT_UREFS_OVERFLOW(urefs+1, delta))
goto urefs_overflow;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
assert((entry->ie_bits & MACH_PORT_TYPE_SEND) == 0);
goto invalid_right;
}
/* port is locked and active */
assert(port->ip_srights > 0);
if ((urefs + delta) == 0) {
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
if (bits & MACH_PORT_TYPE_RECEIVE) {
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK|
MACH_PORT_TYPE_SEND);
} else {
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND);
dnrequest = ipc_right_dncancel_macro(
space, port, name, entry);
ipc_reverse_remove(space, (ipc_object_t) port);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
ip_release(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
}
} else
entry->ie_bits = bits + delta;
ip_unlock(port); /* even if dropped a ref, port is active */
is_write_unlock(space);
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_delta: strange right");
#else
panic("ipc_right_delta: strange right");
#endif
}
return KERN_SUCCESS;
success:
is_write_unlock(space);
return KERN_SUCCESS;
invalid_right:
is_write_unlock(space);
return KERN_INVALID_RIGHT;
invalid_value:
is_write_unlock(space);
return KERN_INVALID_VALUE;
urefs_overflow:
is_write_unlock(space);
return KERN_UREFS_OVERFLOW;
}
kern_return_t
ipc_right_destroy(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry)
{
ipc_entry_bits_t bits = entry->ie_bits;
mach_port_type_t type = IE_BITS_TYPE(bits);
assert(space->is_active);
switch (type) {
case MACH_PORT_TYPE_DEAD_NAME:
assert(entry->ie_request == 0);
assert(entry->ie_object == IO_NULL);
assert((bits & IE_BITS_MAREQUEST) == 0);
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
break;
case MACH_PORT_TYPE_PORT_SET: {
ipc_pset_t pset = (ipc_pset_t) entry->ie_object;
assert(entry->ie_request == 0);
assert(pset != IPS_NULL);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
ips_lock(pset);
assert(ips_active(pset));
is_write_unlock(space);
ipc_pset_destroy(pset); /* consumes ref, unlocks */
break;
}
case MACH_PORT_TYPE_SEND:
case MACH_PORT_TYPE_RECEIVE:
case MACH_PORT_TYPE_SEND_RECEIVE:
case MACH_PORT_TYPE_SEND_ONCE: {
ipc_port_t port = (ipc_port_t) entry->ie_object;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
ipc_port_t dnrequest;
assert(port != IP_NULL);
if (bits & IE_BITS_MAREQUEST) {
assert(type & MACH_PORT_TYPE_SEND_RECEIVE);
ipc_marequest_cancel(space, name);
}
if (type == MACH_PORT_TYPE_SEND)
ipc_reverse_remove(space, (ipc_object_t) port);
ip_lock(port);
if (!ip_active(port)) {
assert((type & MACH_PORT_TYPE_RECEIVE) == 0);
ip_release(port);
ip_check_unlock(port);
entry->ie_request = 0;
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
break;
}
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
if (type & MACH_PORT_TYPE_SEND) {
assert(port->ip_srights > 0);
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
}
if (type & MACH_PORT_TYPE_RECEIVE) {
assert(ip_active(port));
assert(port->ip_receiver == space);
ipc_port_clear_receiver(port);
ipc_port_destroy(port); /* consumes our ref, unlocks */
} else if (type & MACH_PORT_TYPE_SEND_ONCE) {
assert(port->ip_sorights > 0);
ip_unlock(port);
ipc_notify_send_once(port); /* consumes our ref */
} else {
assert(port->ip_receiver != space);
ip_release(port);
ip_unlock(port);
}
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_destroy: strange type");
#else
panic("ipc_right_destroy: strange type");
#endif
}
return KERN_SUCCESS;
}
void
ipc_right_clean(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry)
{
ipc_entry_bits_t bits = entry->ie_bits;
mach_port_type_t type = IE_BITS_TYPE(bits);
assert(!space->is_active);
/*
* We can't clean up IE_BITS_MAREQUEST when the space is dead.
* This is because ipc_marequest_destroy can't turn off
* the bit if the space is dead. Hence, it might be on
* even though the marequest has been destroyed. It's OK
* not to cancel the marequest, because ipc_marequest_destroy
* cancels for us if the space is dead.
*
* IE_BITS_COMPAT/ipc_right_dncancel doesn't have this
* problem, because we check that the port is active. If
* we didn't cancel IE_BITS_COMPAT, ipc_port_destroy
* would still work, but dead space refs would accumulate
* in ip_dnrequests. They would use up slots in
* ip_dnrequests and keep the spaces from being freed.
*/
switch (type) {
case MACH_PORT_TYPE_DEAD_NAME:
assert(entry->ie_request == 0);
assert(entry->ie_object == IO_NULL);
assert((bits & IE_BITS_MAREQUEST) == 0);
break;
case MACH_PORT_TYPE_PORT_SET: {
ipc_pset_t pset = (ipc_pset_t) entry->ie_object;
assert(entry->ie_request == 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(pset != IPS_NULL);
ips_lock(pset);
assert(ips_active(pset));
ipc_pset_destroy(pset); /* consumes ref, unlocks */
break;
}
case MACH_PORT_TYPE_SEND:
case MACH_PORT_TYPE_RECEIVE:
case MACH_PORT_TYPE_SEND_RECEIVE:
case MACH_PORT_TYPE_SEND_ONCE: {
ipc_port_t port = (ipc_port_t) entry->ie_object;
ipc_port_t dnrequest;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
assert(port != IP_NULL);
ip_lock(port);
if (!ip_active(port)) {
ip_release(port);
ip_check_unlock(port);
break;
}
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
if (type & MACH_PORT_TYPE_SEND) {
assert(port->ip_srights > 0);
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
}
if (type & MACH_PORT_TYPE_RECEIVE) {
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
ipc_port_clear_receiver(port);
ipc_port_destroy(port); /* consumes our ref, unlocks */
} else if (type & MACH_PORT_TYPE_SEND_ONCE) {
assert(port->ip_sorights > 0);
ip_unlock(port);
ipc_notify_send_once(port); /* consumes our ref */
} else {
assert(port->ip_receiver != space);
ip_release(port);
ip_unlock(port); /* port is active */
}
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_clean: strange type");
#else
panic("ipc_right_clean: strange type");
#endif
}
}
kern_return_t
ipc_right_copyout(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry,
mach_msg_type_name_t msgt_name,
boolean_t overflow,
ipc_object_t object)
{
ipc_entry_bits_t bits = entry->ie_bits;
ipc_port_t port;
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
assert(io_active(object));
assert(entry->ie_object == object);
port = (ipc_port_t) object;
switch (msgt_name) {
case MACH_MSG_TYPE_PORT_SEND_ONCE:
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE);
assert(port->ip_sorights > 0);
/* transfer send-once right and ref to entry */
ip_unlock(port);
entry->ie_bits = bits | (MACH_PORT_TYPE_SEND_ONCE | 1);
break;
case MACH_MSG_TYPE_PORT_SEND:
assert(port->ip_srights > 0);
if (bits & MACH_PORT_TYPE_SEND) {
mach_port_urefs_t urefs = IE_BITS_UREFS(bits);
assert(port->ip_srights > 1);
assert(urefs > 0);
assert(urefs < MACH_PORT_UREFS_MAX);
if (urefs+1 == MACH_PORT_UREFS_MAX) {
if (overflow) {
/* leave urefs pegged to maximum */
port->ip_srights--;
ip_release(port);
ip_unlock(port);
return KERN_SUCCESS;
}
ip_unlock(port);
return KERN_UREFS_OVERFLOW;
}
port->ip_srights--;
ip_release(port);
ip_unlock(port);
} else if (bits & MACH_PORT_TYPE_RECEIVE) {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE);
assert(IE_BITS_UREFS(bits) == 0);
/* transfer send right to entry */
ip_release(port);
ip_unlock(port);
} else {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE);
assert(IE_BITS_UREFS(bits) == 0);
/* transfer send right and ref to entry */
ip_unlock(port);
/* entry is locked holding ref, so can use port */
entry->ie_name = name;
ipc_reverse_insert(space, (ipc_object_t) port, entry);
}
entry->ie_bits = (bits | MACH_PORT_TYPE_SEND) + 1;
break;
case MACH_MSG_TYPE_PORT_RECEIVE: {
ipc_port_t dest;
assert(port->ip_mscount == 0);
assert(port->ip_receiver_name == MACH_PORT_NULL);
dest = port->ip_destination;
port->ip_receiver_name = name;
port->ip_receiver = space;
/*
* Clear the protected payload field to retain
* the behavior of mach_msg.
*/
ipc_port_flag_protected_payload_clear(port);
assert((bits & MACH_PORT_TYPE_RECEIVE) == 0);
if (bits & MACH_PORT_TYPE_SEND) {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND);
assert(IE_BITS_UREFS(bits) > 0);
assert(port->ip_srights > 0);
ip_release(port);
ip_unlock(port);
/* entry is locked holding ref, so can use port */
ipc_reverse_remove(space, (ipc_object_t) port);
} else {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE);
assert(IE_BITS_UREFS(bits) == 0);
/* transfer ref to entry */
ip_unlock(port);
}
entry->ie_bits = bits | MACH_PORT_TYPE_RECEIVE;
if (dest != IP_NULL)
ipc_port_release(dest);
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_copyout: strange rights");
#else
panic("ipc_right_copyout: strange rights");
#endif
}
return KERN_SUCCESS;
}
void
ipc_port_destroy(
ipc_port_t port)
{
ipc_port_t pdrequest, nsrequest;
ipc_mqueue_t mqueue;
ipc_kmsg_t kmsg;
ipc_port_request_t dnrequests;
assert(ip_active(port));
/* port->ip_receiver_name is garbage */
/* port->ip_receiver/port->ip_destination is garbage */
assert(port->ip_pset_count == 0);
assert(port->ip_mscount == 0);
/* first check for a backup port */
pdrequest = port->ip_pdrequest;
if (pdrequest != IP_NULL) {
/* we assume the ref for pdrequest */
port->ip_pdrequest = IP_NULL;
/* make port be in limbo */
port->ip_receiver_name = MACH_PORT_NULL;
port->ip_destination = IP_NULL;
ip_unlock(port);
/* consumes our refs for port and pdrequest */
ipc_notify_port_destroyed(pdrequest, port);
return;
}
/* once port is dead, we don't need to keep it locked */
port->ip_object.io_bits &= ~IO_BITS_ACTIVE;
port->ip_timestamp = ipc_port_timestamp();
/* save for later */
dnrequests = port->ip_dnrequests;
port->ip_dnrequests = IPR_NULL;
/*
* If the port has a preallocated message buffer and that buffer
* is not inuse, free it. If it has an inuse one, then the kmsg
* free will detect that we freed the association and it can free it
* like a normal buffer.
*/
if (IP_PREALLOC(port)) {
kmsg = port->ip_premsg;
assert(kmsg != IKM_NULL);
IP_CLEAR_PREALLOC(port, kmsg);
if (!ikm_prealloc_inuse(kmsg))
ipc_kmsg_free(kmsg);
}
ip_unlock(port);
/* throw away no-senders request */
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL)
ipc_notify_send_once(nsrequest); /* consumes ref */
/* destroy any queued messages */
mqueue = &port->ip_messages;
ipc_mqueue_destroy(mqueue);
/* generate dead-name notifications */
if (dnrequests != IPR_NULL) {
ipc_port_dnnotify(port, dnrequests);
}
ipc_kobject_destroy(port);
ipc_port_release(port); /* consume caller's ref */
}
void
ipc_object_copyin_from_kernel(
ipc_object_t object,
mach_msg_type_name_t msgt_name)
{
assert(IO_VALID(object));
switch (msgt_name) {
case MACH_MSG_TYPE_MOVE_RECEIVE: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
assert(ip_active(port));
if (port->ip_destination != IP_NULL) {
assert(port->ip_receiver == ipc_space_kernel);
/* relevant part of ipc_port_clear_receiver */
ipc_port_set_mscount(port, 0);
port->ip_receiver_name = MACH_PORT_NULL;
port->ip_destination = IP_NULL;
}
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_COPY_SEND: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
if (ip_active(port)) {
assert(port->ip_srights > 0);
port->ip_srights++;
}
ip_reference(port);
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_MAKE_SEND: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
if (ip_active(port)) {
assert(port->ip_receiver_name != MACH_PORT_NULL);
assert((port->ip_receiver == ipc_space_kernel) ||
(port->ip_receiver->is_node_id != HOST_LOCAL_NODE));
port->ip_mscount++;
}
port->ip_srights++;
ip_reference(port);
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_MOVE_SEND: {
/* move naked send right into the message */
assert(((ipc_port_t)object)->ip_srights);
break;
}
case MACH_MSG_TYPE_MAKE_SEND_ONCE: {
ipc_port_t port = (ipc_port_t) object;
ip_lock(port);
if (ip_active(port)) {
assert(port->ip_receiver_name != MACH_PORT_NULL);
}
port->ip_sorights++;
ip_reference(port);
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_MOVE_SEND_ONCE: {
/* move naked send-once right into the message */
assert(((ipc_port_t)object)->ip_sorights);
break;
}
default:
panic("ipc_object_copyin_from_kernel: strange rights");
}
}
boolean_t
ipc_port_check_circularity(
ipc_port_t port,
ipc_port_t dest)
{
ipc_port_t base;
assert(port != IP_NULL);
assert(dest != IP_NULL);
if (port == dest)
return TRUE;
base = dest;
/*
* First try a quick check that can run in parallel.
* No circularity if dest is not in transit.
*/
ip_lock(port);
if (ip_lock_try(dest)) {
if (!ip_active(dest) ||
(dest->ip_receiver_name != MACH_PORT_NULL) ||
(dest->ip_destination == IP_NULL))
goto not_circular;
/* dest is in transit; further checking necessary */
ip_unlock(dest);
}
ip_unlock(port);
ipc_port_multiple_lock(); /* massive serialization */
/*
* Search for the end of the chain (a port not in transit),
* acquiring locks along the way.
*/
for (;;) {
ip_lock(base);
if (!ip_active(base) ||
(base->ip_receiver_name != MACH_PORT_NULL) ||
(base->ip_destination == IP_NULL))
break;
base = base->ip_destination;
}
/* all ports in chain from dest to base, inclusive, are locked */
if (port == base) {
/* circularity detected! */
ipc_port_multiple_unlock();
/* port (== base) is in limbo */
assert(ip_active(port));
assert(port->ip_receiver_name == MACH_PORT_NULL);
assert(port->ip_destination == IP_NULL);
while (dest != IP_NULL) {
ipc_port_t next;
/* dest is in transit or in limbo */
assert(ip_active(dest));
assert(dest->ip_receiver_name == MACH_PORT_NULL);
next = dest->ip_destination;
ip_unlock(dest);
dest = next;
}
return TRUE;
}
/*
* The guarantee: lock port while the entire chain is locked.
* Once port is locked, we can take a reference to dest,
* add port to the chain, and unlock everything.
*/
ip_lock(port);
ipc_port_multiple_unlock();
not_circular:
/* port is in limbo */
assert(ip_active(port));
assert(port->ip_receiver_name == MACH_PORT_NULL);
assert(port->ip_destination == IP_NULL);
ip_reference(dest);
port->ip_destination = dest;
/* now unlock chain */
while (port != base) {
ipc_port_t next;
/* port is in transit */
assert(ip_active(port));
assert(port->ip_receiver_name == MACH_PORT_NULL);
assert(port->ip_destination != IP_NULL);
next = port->ip_destination;
ip_unlock(port);
port = next;
}
/* base is not in transit */
assert(!ip_active(base) ||
(base->ip_receiver_name != MACH_PORT_NULL) ||
(base->ip_destination == IP_NULL));
ip_unlock(base);
return FALSE;
}
boolean_t
ipc_port_check_circularity(
ipc_port_t port,
ipc_port_t dest)
{
ipc_port_t base;
#if IMPORTANCE_INHERITANCE
task_t task = TASK_NULL;
task_t release_task = TASK_NULL;
int assertcnt = 0;
#endif /* IMPORTANCE_INHERITANCE */
assert(port != IP_NULL);
assert(dest != IP_NULL);
if (port == dest)
return TRUE;
base = dest;
/*
* First try a quick check that can run in parallel.
* No circularity if dest is not in transit.
*/
ip_lock(port);
if (ip_lock_try(dest)) {
if (!ip_active(dest) ||
(dest->ip_receiver_name != MACH_PORT_NULL) ||
(dest->ip_destination == IP_NULL))
goto not_circular;
/* dest is in transit; further checking necessary */
ip_unlock(dest);
}
ip_unlock(port);
ipc_port_multiple_lock(); /* massive serialization */
/*
* Search for the end of the chain (a port not in transit),
* acquiring locks along the way.
*/
for (;;) {
ip_lock(base);
if (!ip_active(base) ||
(base->ip_receiver_name != MACH_PORT_NULL) ||
(base->ip_destination == IP_NULL))
break;
base = base->ip_destination;
}
/* all ports in chain from dest to base, inclusive, are locked */
if (port == base) {
/* circularity detected! */
ipc_port_multiple_unlock();
/* port (== base) is in limbo */
assert(ip_active(port));
assert(port->ip_receiver_name == MACH_PORT_NULL);
assert(port->ip_destination == IP_NULL);
while (dest != IP_NULL) {
ipc_port_t next;
/* dest is in transit or in limbo */
assert(ip_active(dest));
assert(dest->ip_receiver_name == MACH_PORT_NULL);
next = dest->ip_destination;
ip_unlock(dest);
dest = next;
}
return TRUE;
}
/*
* The guarantee: lock port while the entire chain is locked.
* Once port is locked, we can take a reference to dest,
* add port to the chain, and unlock everything.
*/
ip_lock(port);
ipc_port_multiple_unlock();
not_circular:
/* port is in limbo */
assert(ip_active(port));
assert(port->ip_receiver_name == MACH_PORT_NULL);
assert(port->ip_destination == IP_NULL);
ip_reference(dest);
port->ip_destination = dest;
#if IMPORTANCE_INHERITANCE
/* must have been in limbo or still bound to a task */
assert(port->ip_tempowner != 0);
if (port->ip_taskptr != 0) {
/*
* We delayed dropping assertions from a specific task.
* Cache that info now (we'll drop assertions and the
* task reference below).
*/
release_task = port->ip_imp_task;
port->ip_imp_task = TASK_NULL;
port->ip_taskptr = 0;
}
assertcnt = port->ip_impcount;
/* take the port out of limbo w.r.t. assertions */
port->ip_tempowner = 0;
#endif /* IMPORTANCE_INHERITANCE */
/* now unlock chain */
ip_unlock(port);
for (;;) {
#if IMPORTANCE_INHERITANCE
/* every port along chain track assertions behind it */
dest->ip_impcount += assertcnt;
#endif /* IMPORTANCE_INHERITANCE */
if (dest == base)
break;
/* port is in transit */
assert(ip_active(dest));
assert(dest->ip_receiver_name == MACH_PORT_NULL);
assert(dest->ip_destination != IP_NULL);
#if IMPORTANCE_INHERITANCE
assert(dest->ip_tempowner == 0);
#endif /* IMPORTANCE_INHERITANCE */
port = dest->ip_destination;
ip_unlock(dest);
dest = port;
}
/* base is not in transit */
assert(!ip_active(base) ||
(base->ip_receiver_name != MACH_PORT_NULL) ||
(base->ip_destination == IP_NULL));
#if IMPORTANCE_INHERITANCE
/*
* Find the task to boost (if any).
* We will boost "through" ports that don't know
* about inheritance to deliver receive rights that
* do.
*/
if (ip_active(base) && (assertcnt > 0)) {
if (base->ip_tempowner != 0) {
if (base->ip_taskptr != 0)
/* specified tempowner task */
task = base->ip_imp_task;
/* otherwise don't boost current task */
} else if (base->ip_receiver_name != MACH_PORT_NULL) {
ipc_space_t space = base->ip_receiver;
/* only spaces with boost-accepting tasks */
if (space->is_task != TASK_NULL &&
space->is_task->imp_receiver != 0)
task = space->is_task;
}
/* take reference before unlocking base */
if (task != TASK_NULL) {
assert(task->imp_receiver != 0);
task_reference(task);
}
}
#endif /* IMPORTANCE_INHERITANCE */
ip_unlock(base);
#if IMPORTANCE_INHERITANCE
/*
* Transfer assertions now that the ports are unlocked.
* Avoid extra overhead if transferring to/from the same task.
*/
boolean_t transfer_assertions = (task != release_task) ? TRUE : FALSE;
if (task != TASK_NULL) {
if (transfer_assertions)
task_importance_hold_internal_assertion(task, assertcnt);
task_deallocate(task);
task = TASK_NULL;
}
if (release_task != TASK_NULL) {
if (transfer_assertions)
task_importance_drop_internal_assertion(release_task, assertcnt);
task_deallocate(release_task);
release_task = TASK_NULL;
}
#endif /* IMPORTANCE_INHERITANCE */
return FALSE;
}
void
ipc_port_destroy(
ipc_port_t port)
{
ipc_port_t pdrequest, nsrequest;
ipc_mqueue_t mqueue;
ipc_kmsg_queue_t kmqueue;
ipc_kmsg_t kmsg;
ipc_thread_t sender;
ipc_port_request_t dnrequests;
thread_pool_t thread_pool;
assert(ip_active(port));
/* port->ip_receiver_name is garbage */
/* port->ip_receiver/port->ip_destination is garbage */
assert(io_otype((ipc_object_t)port) == IOT_PORT);
assert(port->ip_pset == IPS_NULL);
assert(port->ip_mscount == 0);
assert(port->ip_seqno == 0);
/* first check for a backup port */
pdrequest = port->ip_pdrequest;
if (pdrequest != IP_NULL) {
/* we assume the ref for pdrequest */
port->ip_pdrequest = IP_NULL;
/* make port be in limbo */
port->ip_receiver_name = MACH_PORT_NAME_NULL;
port->ip_destination = IP_NULL;
ip_unlock(port);
if (!ipc_port_check_circularity(port, pdrequest)) {
/* consumes our refs for port and pdrequest */
ipc_notify_port_destroyed(pdrequest, port);
return;
} else {
/* consume pdrequest and destroy port */
ipc_port_release_sonce(pdrequest);
}
ip_lock(port);
assert(ip_active(port));
assert(port->ip_pset == IPS_NULL);
assert(port->ip_mscount == 0);
assert(port->ip_seqno == 0);
assert(port->ip_pdrequest == IP_NULL);
assert(port->ip_receiver_name == MACH_PORT_NAME_NULL);
assert(port->ip_destination == IP_NULL);
/* fall through and destroy the port */
}
/*
* rouse all blocked senders
*
* This must be done with the port locked, because
* ipc_mqueue_send can play with the ip_blocked queue
* of a dead port.
*/
while ((sender = ipc_thread_dequeue(&port->ip_blocked)) != ITH_NULL) {
sender->ith_state = MACH_MSG_SUCCESS;
thread_go(sender);
}
/* once port is dead, we don't need to keep it locked */
port->ip_object.io_bits &= ~IO_BITS_ACTIVE;
port->ip_timestamp = ipc_port_timestamp();
/* save for later */
dnrequests = port->ip_dnrequests;
port->ip_dnrequests = IPR_NULL;
ip_unlock(port);
/* wakeup any threads waiting on this pool port for an activation */
if ((thread_pool = &port->ip_thread_pool) != THREAD_POOL_NULL)
thread_pool_wakeup(thread_pool);
/* throw away no-senders request */
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL)
ipc_notify_send_once(nsrequest); /* consumes ref */
/* destroy any queued messages */
mqueue = &port->ip_messages;
kmqueue = &mqueue->imq_messages;
while ((kmsg = ipc_kmsg_dequeue(kmqueue)) != IKM_NULL) {
assert(kmsg->ikm_header->msgh_remote_port ==
(mach_port_t) port);
port->ip_msgcount--;
ipc_port_release(port);
kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
ipc_kmsg_destroy(kmsg);
}
/* generate dead-name notifications */
if (dnrequests != IPR_NULL) {
ipc_port_dnnotify(port, dnrequests);
}
if (ip_kotype(port) != IKOT_NONE)
ipc_kobject_destroy(port);
/* XXXX Perhaps should verify that ip_thread_pool is empty! */
ipc_port_release(port); /* consume caller's ref */
}
kern_return_t
ipc_right_copyin_two(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry,
ipc_object_t *objectp,
ipc_port_t *sorightp)
{
ipc_entry_bits_t bits = entry->ie_bits;
mach_port_urefs_t urefs;
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
assert(space->is_active);
if ((bits & MACH_PORT_TYPE_SEND) == 0)
goto invalid_right;
urefs = IE_BITS_UREFS(bits);
if (urefs < 2)
goto invalid_right;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
goto invalid_right;
}
/* port is locked and active */
assert(port->ip_srights > 0);
if (urefs == 2) {
if (bits & MACH_PORT_TYPE_RECEIVE) {
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
port->ip_srights++;
ip_reference(port);
ip_reference(port);
} else {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND);
dnrequest = ipc_right_dncancel_macro(space, port,
name, entry);
ipc_reverse_remove(space, (ipc_object_t) port);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
port->ip_srights++;
ip_reference(port);
entry->ie_object = IO_NULL;
}
entry->ie_bits = bits &~
(IE_BITS_UREFS_MASK|MACH_PORT_TYPE_SEND);
} else {
port->ip_srights += 2;
ip_reference(port);
ip_reference(port);
entry->ie_bits = bits-2; /* decrement urefs */
}
ip_unlock(port);
*objectp = (ipc_object_t) port;
*sorightp = dnrequest;
return KERN_SUCCESS;
invalid_right:
return KERN_INVALID_RIGHT;
}
void
ipc_object_copyout_dest(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
mach_port_name_t *namep)
{
mach_port_name_t name;
assert(IO_VALID(object));
assert(io_active(object));
io_release(object);
/*
* If the space is the receiver/owner of the object,
* then we quietly consume the right and return
* the space's name for the object. Otherwise
* we destroy the right and return MACH_PORT_NULL.
*/
switch (msgt_name) {
case MACH_MSG_TYPE_PORT_SEND: {
ipc_port_t port = (ipc_port_t) object;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount;
if (port->ip_receiver == space)
name = port->ip_receiver_name;
else
name = MACH_PORT_NULL;
assert(port->ip_srights > 0);
if (--port->ip_srights == 0 &&
port->ip_nsrequest != IP_NULL) {
nsrequest = port->ip_nsrequest;
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
ip_unlock(port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(port);
break;
}
case MACH_MSG_TYPE_PORT_SEND_ONCE: {
ipc_port_t port = (ipc_port_t) object;
assert(port->ip_sorights > 0);
if (port->ip_receiver == space) {
/* quietly consume the send-once right */
port->ip_sorights--;
name = port->ip_receiver_name;
ip_unlock(port);
} else {
/*
* A very bizarre case. The message
* was received, but before this copyout
* happened the space lost receive rights.
* We can't quietly consume the soright
* out from underneath some other task,
* so generate a send-once notification.
*/
ip_reference(port); /* restore ref */
ip_unlock(port);
ipc_notify_send_once(port);
name = MACH_PORT_NULL;
}
break;
}
default:
panic("ipc_object_copyout_dest: strange rights");
name = MACH_PORT_DEAD;
}
*namep = name;
}
kern_return_t
mach_port_space_info(
ipc_space_t space,
ipc_info_space_t *infop,
ipc_info_name_array_t *tablep,
mach_msg_type_number_t *tableCntp,
__unused ipc_info_tree_name_array_t *treep,
__unused mach_msg_type_number_t *treeCntp)
{
ipc_info_name_t *table_info;
vm_offset_t table_addr;
vm_size_t table_size, table_size_needed;
ipc_entry_t table;
ipc_entry_num_t tsize;
mach_port_index_t index;
kern_return_t kr;
vm_map_copy_t copy;
if (space == IS_NULL)
return KERN_INVALID_TASK;
#if !(DEVELOPMENT | DEBUG)
const boolean_t dbg_ok = (mac_task_check_expose_task(kernel_task) == 0);
#else
const boolean_t dbg_ok = TRUE;
#endif
/* start with in-line memory */
table_size = 0;
for (;;) {
is_read_lock(space);
if (!is_active(space)) {
is_read_unlock(space);
if (table_size != 0)
kmem_free(ipc_kernel_map,
table_addr, table_size);
return KERN_INVALID_TASK;
}
table_size_needed =
vm_map_round_page((space->is_table_size
* sizeof(ipc_info_name_t)),
VM_MAP_PAGE_MASK(ipc_kernel_map));
if (table_size_needed == table_size)
break;
is_read_unlock(space);
if (table_size != table_size_needed) {
if (table_size != 0)
kmem_free(ipc_kernel_map, table_addr, table_size);
kr = kmem_alloc(ipc_kernel_map, &table_addr, table_size_needed, VM_KERN_MEMORY_IPC);
if (kr != KERN_SUCCESS) {
return KERN_RESOURCE_SHORTAGE;
}
table_size = table_size_needed;
}
}
/* space is read-locked and active; we have enough wired memory */
/* get the overall space info */
infop->iis_genno_mask = MACH_PORT_NGEN(MACH_PORT_DEAD);
infop->iis_table_size = space->is_table_size;
infop->iis_table_next = space->is_table_next->its_size;
/* walk the table for this space */
table = space->is_table;
tsize = space->is_table_size;
table_info = (ipc_info_name_array_t)table_addr;
for (index = 0; index < tsize; index++) {
ipc_info_name_t *iin = &table_info[index];
ipc_entry_t entry = &table[index];
ipc_entry_bits_t bits;
bits = entry->ie_bits;
iin->iin_name = MACH_PORT_MAKE(index, IE_BITS_GEN(bits));
iin->iin_collision = 0;
iin->iin_type = IE_BITS_TYPE(bits);
if ((entry->ie_bits & MACH_PORT_TYPE_PORT_RIGHTS) != MACH_PORT_TYPE_NONE &&
entry->ie_request != IE_REQ_NONE) {
__IGNORE_WCASTALIGN(ipc_port_t port = (ipc_port_t) entry->ie_object);
assert(IP_VALID(port));
ip_lock(port);
iin->iin_type |= ipc_port_request_type(port, iin->iin_name, entry->ie_request);
ip_unlock(port);
}
iin->iin_urefs = IE_BITS_UREFS(bits);
iin->iin_object = (dbg_ok) ? (natural_t)VM_KERNEL_ADDRPERM((uintptr_t)entry->ie_object) : 0;
iin->iin_next = entry->ie_next;
iin->iin_hash = entry->ie_index;
}
is_read_unlock(space);
/* prepare the table out-of-line data for return */
if (table_size > 0) {
vm_size_t used_table_size;
used_table_size = infop->iis_table_size * sizeof(ipc_info_name_t);
if (table_size > used_table_size)
bzero((char *)&table_info[infop->iis_table_size],
table_size - used_table_size);
kr = vm_map_unwire(
ipc_kernel_map,
vm_map_trunc_page(table_addr,
VM_MAP_PAGE_MASK(ipc_kernel_map)),
vm_map_round_page(table_addr + table_size,
VM_MAP_PAGE_MASK(ipc_kernel_map)),
FALSE);
assert(kr == KERN_SUCCESS);
kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)table_addr,
(vm_map_size_t)used_table_size, TRUE, ©);
assert(kr == KERN_SUCCESS);
*tablep = (ipc_info_name_t *)copy;
*tableCntp = infop->iis_table_size;
} else {
*tablep = (ipc_info_name_t *)0;
*tableCntp = 0;
}
/* splay tree is obsolete, no work to do... */
*treep = (ipc_info_tree_name_t *)0;
*treeCntp = 0;
return KERN_SUCCESS;
}
boolean_t
ipc_port_importance_delta(
ipc_port_t port,
mach_port_delta_t delta)
{
ipc_port_t next, base;
task_t task = TASK_NULL;
boolean_t dropped = FALSE;
if (delta == 0)
return FALSE;
base = port;
/* if port is in transit, have to search for end of chain */
if (ip_active(port) &&
port->ip_destination != IP_NULL &&
port->ip_receiver_name == MACH_PORT_NULL) {
dropped = TRUE;
ip_unlock(port);
ipc_port_multiple_lock(); /* massive serialization */
ip_lock(base);
while(ip_active(base) &&
base->ip_destination != IP_NULL &&
base->ip_receiver_name == MACH_PORT_NULL) {
base = base->ip_destination;
ip_lock(base);
}
ipc_port_multiple_unlock();
}
/* unlock down to the base, adding a boost at each level */
for (;;) {
port->ip_impcount += delta;
if (port == base)
break;
/* port is in transit */
assert(port->ip_tempowner == 0);
next = port->ip_destination;
ip_unlock(port);
port = next;
}
/* find the task (if any) to boost according to the base */
if (ip_active(base)) {
if (base->ip_tempowner != 0) {
if (base->ip_taskptr != 0)
task = base->ip_imp_task;
/* otherwise don't boost */
} else if (base->ip_receiver_name != MACH_PORT_NULL) {
ipc_space_t space = base->ip_receiver;
/* only spaces with boost-accepting tasks */
if (space->is_task != TASK_NULL &&
space->is_task->imp_receiver != 0)
task = space->is_task;
}
}
/*
* Only the base is locked. If we have to hold or drop task
* importance assertions, we'll have to drop that lock as well.
*/
if (task != TASK_NULL) {
/* take a reference before unlocking base */
assert(task->imp_receiver != 0);
task_reference(task);
ip_unlock(base);
dropped = TRUE;
if (delta > 0)
task_importance_hold_internal_assertion(task, delta);
else
task_importance_drop_internal_assertion(task, -delta);
task_deallocate(task);
} else if (dropped == TRUE) {
ip_unlock(base);
}
return dropped;
}
/*
* thread_switch:
*
* Context switch. User may supply thread hint.
*/
kern_return_t
thread_switch(
struct thread_switch_args *args)
{
thread_t thread = THREAD_NULL;
thread_t self = current_thread();
mach_port_name_t thread_name = args->thread_name;
int option = args->option;
mach_msg_timeout_t option_time = args->option_time;
uint32_t scale_factor = NSEC_PER_MSEC;
boolean_t reenable_workq_callback = FALSE;
boolean_t depress_option = FALSE;
boolean_t wait_option = FALSE;
/*
* Validate and process option.
*/
switch (option) {
case SWITCH_OPTION_NONE:
workqueue_thread_yielded();
break;
case SWITCH_OPTION_WAIT:
wait_option = TRUE;
workqueue_thread_yielded();
break;
case SWITCH_OPTION_DEPRESS:
depress_option = TRUE;
workqueue_thread_yielded();
break;
case SWITCH_OPTION_DISPATCH_CONTENTION:
scale_factor = NSEC_PER_USEC;
wait_option = TRUE;
if (thread_switch_disable_workqueue_sched_callback())
reenable_workq_callback = TRUE;
break;
case SWITCH_OPTION_OSLOCK_DEPRESS:
depress_option = TRUE;
if (thread_switch_disable_workqueue_sched_callback())
reenable_workq_callback = TRUE;
break;
case SWITCH_OPTION_OSLOCK_WAIT:
wait_option = TRUE;
if (thread_switch_disable_workqueue_sched_callback())
reenable_workq_callback = TRUE;
break;
default:
return (KERN_INVALID_ARGUMENT);
}
/*
* Translate the port name if supplied.
*/
if (thread_name != MACH_PORT_NULL) {
ipc_port_t port;
if (ipc_port_translate_send(self->task->itk_space,
thread_name, &port) == KERN_SUCCESS) {
ip_reference(port);
ip_unlock(port);
thread = convert_port_to_thread(port);
ip_release(port);
if (thread == self) {
thread_deallocate(thread);
thread = THREAD_NULL;
}
}
}
if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
if (thread != THREAD_NULL) {
if (thread->task != self->task) {
/*
* OSLock boosting only applies to other threads
* in your same task (even if you have a port for
* a thread in another task)
*/
thread_deallocate(thread);
thread = THREAD_NULL;
} else {
/*
* Attempt to kick the lock owner up to our same IO throttling tier.
* If the thread is currently blocked in throttle_lowpri_io(),
* it will immediately break out.
*
* TODO: SFI break out?
*/
int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
set_thread_iotier_override(thread, new_policy);
}
}
}
/*
* Try to handoff if supplied.
*/
if (thread != THREAD_NULL) {
spl_t s = splsched();
/* This may return a different thread if the target is pushing on something */
thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_THREAD_SWITCH)|DBG_FUNC_NONE,
thread_tid(thread), thread->state,
pulled_thread ? TRUE : FALSE, 0, 0);
if (pulled_thread != THREAD_NULL) {
/* We can't be dropping the last ref here */
thread_deallocate_safe(thread);
if (wait_option)
assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
option_time, scale_factor);
else if (depress_option)
thread_depress_ms(option_time);
self->saved.swtch.option = option;
self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
thread_run(self, (thread_continue_t)thread_switch_continue, NULL, pulled_thread);
/* NOTREACHED */
panic("returned from thread_run!");
}
splx(s);
thread_deallocate(thread);
}
if (wait_option)
assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor);
else if (depress_option)
thread_depress_ms(option_time);
self->saved.swtch.option = option;
self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
if (depress_option)
thread_depress_abort_internal(self);
if (reenable_workq_callback)
thread_switch_enable_workqueue_sched_callback();
return (KERN_SUCCESS);
}
/*
* Routine: ipc_port_spnotify
* Purpose:
* Generate send-possible port notifications.
* Conditions:
* Nothing locked, reference held on port.
*/
void
ipc_port_spnotify(
ipc_port_t port)
{
ipc_port_request_index_t index = 0;
ipc_table_elems_t size = 0;
#if IMPORTANCE_INHERITANCE
boolean_t dropassert = FALSE;
#endif /* IMPORTANCE_INHERITANCE */
/*
* If the port has no send-possible request
* armed, don't bother to lock the port.
*/
if (port->ip_sprequests == 0)
return;
ip_lock(port);
#if IMPORTANCE_INHERITANCE
if (port->ip_spimportant != 0) {
port->ip_spimportant = 0;
port->ip_impcount--;
dropassert = TRUE;
}
#endif /* IMPORTANCE_INHERITANCE */
if (port->ip_sprequests == 0) {
ip_unlock(port);
goto out;
}
port->ip_sprequests = 0;
revalidate:
if (ip_active(port)) {
ipc_port_request_t requests;
/* table may change each time port unlocked (reload) */
requests = port->ip_requests;
assert(requests != IPR_NULL);
/*
* no need to go beyond table size when first
* we entered - those are future notifications.
*/
if (size == 0)
size = requests->ipr_size->its_size;
/* no need to backtrack either */
while (++index < size) {
ipc_port_request_t ipr = &requests[index];
mach_port_name_t name = ipr->ipr_name;
ipc_port_t soright = IPR_SOR_PORT(ipr->ipr_soright);
boolean_t armed = IPR_SOR_SPARMED(ipr->ipr_soright);
if (MACH_PORT_VALID(name) && armed && IP_VALID(soright)) {
/* claim send-once right - slot still inuse */
ipr->ipr_soright = IP_NULL;
ip_unlock(port);
ipc_notify_send_possible(soright, name);
ip_lock(port);
goto revalidate;
}
}
}
ip_unlock(port);
out:
#if IMPORTANCE_INHERITANCE
if ((dropassert == TRUE) && (current_task()->imp_receiver != 0)) {
/* drop internal assertion and no task lock held */
task_importance_drop_internal_assertion(current_task(), 1);
}
#endif /* IMPORTANCE_INHERITANCE */
return;
}
/*
* Routine: exception_deliver
* Purpose:
* Make an upcall to the exception server provided.
* Conditions:
* Nothing locked and no resources held.
* Called from an exception context, so
* thread_exception_return and thread_kdb_return
* are possible.
* Returns:
* KERN_SUCCESS if the exception was handled
*/
kern_return_t
exception_deliver(
thread_t thread,
exception_type_t exception,
mach_exception_data_t code,
mach_msg_type_number_t codeCnt,
struct exception_action *excp,
lck_mtx_t *mutex)
{
ipc_port_t exc_port;
exception_data_type_t small_code[EXCEPTION_CODE_MAX];
int code64;
int behavior;
int flavor;
kern_return_t kr;
/*
* Save work if we are terminating.
* Just go back to our AST handler.
*/
if (!thread->active)
return KERN_SUCCESS;
/*
* Snapshot the exception action data under lock for consistency.
* Hold a reference to the port over the exception_raise_* calls
* so it can't be destroyed. This seems like overkill, but keeps
* the port from disappearing between now and when
* ipc_object_copyin_from_kernel is finally called.
*/
lck_mtx_lock(mutex);
exc_port = excp->port;
if (!IP_VALID(exc_port)) {
lck_mtx_unlock(mutex);
return KERN_FAILURE;
}
ip_lock(exc_port);
if (!ip_active(exc_port)) {
ip_unlock(exc_port);
lck_mtx_unlock(mutex);
return KERN_FAILURE;
}
ip_reference(exc_port);
exc_port->ip_srights++;
ip_unlock(exc_port);
flavor = excp->flavor;
behavior = excp->behavior;
lck_mtx_unlock(mutex);
code64 = (behavior & MACH_EXCEPTION_CODES);
behavior &= ~MACH_EXCEPTION_CODES;
if (!code64) {
small_code[0] = CAST_DOWN_EXPLICIT(exception_data_type_t, code[0]);
small_code[1] = CAST_DOWN_EXPLICIT(exception_data_type_t, code[1]);
}
switch (behavior) {
case EXCEPTION_STATE: {
mach_msg_type_number_t state_cnt;
thread_state_data_t state;
c_thr_exc_raise_state++;
state_cnt = _MachineStateCount[flavor];
kr = thread_getstatus(thread, flavor,
(thread_state_t)state,
&state_cnt);
if (kr == KERN_SUCCESS) {
if (code64) {
kr = mach_exception_raise_state(exc_port,
exception,
code,
codeCnt,
&flavor,
state, state_cnt,
state, &state_cnt);
} else {
kr = exception_raise_state(exc_port, exception,
small_code,
codeCnt,
&flavor,
state, state_cnt,
state, &state_cnt);
}
if (kr == MACH_MSG_SUCCESS)
kr = thread_setstatus(thread, flavor,
(thread_state_t)state,
state_cnt);
}
return kr;
}
case EXCEPTION_DEFAULT:
c_thr_exc_raise++;
if (code64) {
kr = mach_exception_raise(exc_port,
retrieve_thread_self_fast(thread),
retrieve_task_self_fast(thread->task),
exception,
code,
codeCnt);
} else {
kr = exception_raise(exc_port,
retrieve_thread_self_fast(thread),
retrieve_task_self_fast(thread->task),
exception,
small_code,
codeCnt);
}
return kr;
case EXCEPTION_STATE_IDENTITY: {
mach_msg_type_number_t state_cnt;
thread_state_data_t state;
c_thr_exc_raise_state_id++;
state_cnt = _MachineStateCount[flavor];
kr = thread_getstatus(thread, flavor,
(thread_state_t)state,
&state_cnt);
if (kr == KERN_SUCCESS) {
if (code64) {
kr = mach_exception_raise_state_identity(
exc_port,
retrieve_thread_self_fast(thread),
retrieve_task_self_fast(thread->task),
exception,
code,
codeCnt,
&flavor,
state, state_cnt,
state, &state_cnt);
} else {
kr = exception_raise_state_identity(exc_port,
retrieve_thread_self_fast(thread),
retrieve_task_self_fast(thread->task),
exception,
small_code,
codeCnt,
&flavor,
state, state_cnt,
state, &state_cnt);
}
if (kr == MACH_MSG_SUCCESS)
kr = thread_setstatus(thread, flavor,
(thread_state_t)state,
state_cnt);
}
return kr;
}
default:
panic ("bad exception behavior!");
return KERN_FAILURE;
}/* switch */
}
mach_msg_return_t
mach_msg_trap(
mach_msg_header_t *msg,
mach_msg_option_t option,
mach_msg_size_t send_size,
mach_msg_size_t rcv_size,
mach_port_t rcv_name,
mach_msg_timeout_t time_out,
mach_port_t notify)
{
mach_msg_return_t mr;
/* first check for common cases */
if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) {
ipc_thread_t self = current_thread();
ipc_space_t space = self->task->itk_space;
ipc_kmsg_t kmsg;
ipc_port_t dest_port;
ipc_object_t rcv_object;
ipc_mqueue_t rcv_mqueue;
mach_msg_size_t reply_size;
/*
* This case is divided into ten sections, each
* with a label. There are five optimized
* sections and six unoptimized sections, which
* do the same thing but handle all possible
* cases and are slower.
*
* The five sections for an RPC are
* 1) Get request message into a buffer.
* (fast_get or slow_get)
* 2) Copyin request message and rcv_name.
* (fast_copyin or slow_copyin)
* 3) Enqueue request and dequeue reply.
* (fast_send_receive or
* slow_send and slow_receive)
* 4) Copyout reply message.
* (fast_copyout or slow_copyout)
* 5) Put reply message to user's buffer.
* (fast_put or slow_put)
*
* Keep the locking hierarchy firmly in mind.
* (First spaces, then ports, then port sets,
* then message queues.) Only a non-blocking
* attempt can be made to acquire locks out of
* order, or acquire two locks on the same level.
* Acquiring two locks on the same level will
* fail if the objects are really the same,
* unless simple locking is disabled. This is OK,
* because then the extra unlock does nothing.
*
* There are two major reasons these RPCs can't use
* ipc_thread_switch, and use slow_send/slow_receive:
* 1) Kernel RPCs.
* 2) Servers fall behind clients, so
* client doesn't find a blocked server thread and
* server finds waiting messages and can't block.
*/
/*
fast_get:
*/
/*
* optimized ipc_kmsg_get
*
* No locks, references, or messages held.
* We must clear ikm_cache before copyinmsg.
*/
if ((send_size > IKM_SAVED_MSG_SIZE) ||
(send_size < sizeof(mach_msg_header_t)) ||
(send_size & 3) ||
((kmsg = ikm_cache()) == IKM_NULL))
goto slow_get;
ikm_cache() = IKM_NULL;
ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
if (copyinmsg(msg, &kmsg->ikm_header,
send_size)) {
ikm_free(kmsg);
goto slow_get;
}
kmsg->ikm_header.msgh_size = send_size;
fast_copyin:
/*
* optimized ipc_kmsg_copyin/ipc_mqueue_copyin
*
* We have the request message data in kmsg.
* Must still do copyin, send, receive, etc.
*
* If the message isn't simple, we can't combine
* ipc_kmsg_copyin_header and ipc_mqueue_copyin,
* because copyin of the message body might
* affect rcv_name.
*/
switch (kmsg->ikm_header.msgh_bits) {
case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
MACH_MSG_TYPE_MAKE_SEND_ONCE): {
ipc_entry_t table;
ipc_entry_num_t size;
ipc_port_t reply_port;
/* sending a request message */
{
mach_port_index_t index;
mach_port_gen_t gen;
{
mach_port_t reply_name =
kmsg->ikm_header.msgh_local_port;
if (reply_name != rcv_name)
goto slow_copyin;
/* optimized ipc_entry_lookup of reply_name */
index = MACH_PORT_INDEX(reply_name);
gen = MACH_PORT_GEN(reply_name);
}
is_read_lock(space);
assert(space->is_active);
size = space->is_table_size;
table = space->is_table;
if (index >= size)
goto abort_request_copyin;
{
ipc_entry_t entry;
ipc_entry_bits_t bits;
entry = &table[index];
bits = entry->ie_bits;
/* check generation number and type bit */
if ((bits & (IE_BITS_GEN_MASK|
MACH_PORT_TYPE_RECEIVE)) !=
(gen | MACH_PORT_TYPE_RECEIVE))
goto abort_request_copyin;
reply_port = (ipc_port_t) entry->ie_object;
assert(reply_port != IP_NULL);
}
}
/* optimized ipc_entry_lookup of dest_name */
{
mach_port_index_t index;
mach_port_gen_t gen;
{
mach_port_t dest_name =
kmsg->ikm_header.msgh_remote_port;
index = MACH_PORT_INDEX(dest_name);
gen = MACH_PORT_GEN(dest_name);
}
if (index >= size)
goto abort_request_copyin;
{
ipc_entry_t entry;
ipc_entry_bits_t bits;
entry = &table[index];
bits = entry->ie_bits;
/* check generation number and type bit */
if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
(gen | MACH_PORT_TYPE_SEND))
goto abort_request_copyin;
assert(IE_BITS_UREFS(bits) > 0);
dest_port = (ipc_port_t) entry->ie_object;
assert(dest_port != IP_NULL);
}
}
/*
* To do an atomic copyin, need simultaneous
* locks on both ports and the space. If
* dest_port == reply_port, and simple locking is
* enabled, then we will abort. Otherwise it's
* OK to unlock twice.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
ip_unlock(dest_port);
goto abort_request_copyin;
}
is_read_unlock(space);
assert(dest_port->ip_srights > 0);
dest_port->ip_srights++;
ip_reference(dest_port);
assert(ip_active(reply_port));
assert(reply_port->ip_receiver_name ==
kmsg->ikm_header.msgh_local_port);
assert(reply_port->ip_receiver == space);
reply_port->ip_sorights++;
ip_reference(reply_port);
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE);
kmsg->ikm_header.msgh_remote_port =
(mach_port_t) dest_port;
kmsg->ikm_header.msgh_local_port =
(mach_port_t) reply_port;
/* make sure we can queue to the destination */
if (dest_port->ip_receiver == ipc_space_kernel) {
/*
* The kernel server has a reference to
* the reply port, which it hands back
* to us in the reply message. We do
* not need to keep another reference to
* it.
*/
ip_unlock(reply_port);
assert(ip_active(dest_port));
ip_unlock(dest_port);
goto kernel_send;
}
if (dest_port->ip_msgcount >= dest_port->ip_qlimit)
goto abort_request_send_receive;
/* optimized ipc_mqueue_copyin */
if (reply_port->ip_pset != IPS_NULL)
goto abort_request_send_receive;
rcv_object = (ipc_object_t) reply_port;
io_reference(rcv_object);
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
io_unlock(rcv_object);
goto fast_send_receive;
abort_request_copyin:
is_read_unlock(space);
goto slow_copyin;
abort_request_send_receive:
ip_unlock(dest_port);
ip_unlock(reply_port);
goto slow_send;
}
case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
ipc_entry_num_t size;
ipc_entry_t table;
/* sending a reply message */
{
mach_port_t reply_name =
kmsg->ikm_header.msgh_local_port;
if (reply_name != MACH_PORT_NULL)
goto slow_copyin;
}
is_write_lock(space);
assert(space->is_active);
/* optimized ipc_entry_lookup */
size = space->is_table_size;
table = space->is_table;
{
ipc_entry_t entry;
mach_port_gen_t gen;
mach_port_index_t index;
{
mach_port_t dest_name =
kmsg->ikm_header.msgh_remote_port;
index = MACH_PORT_INDEX(dest_name);
gen = MACH_PORT_GEN(dest_name);
}
if (index >= size)
goto abort_reply_dest_copyin;
entry = &table[index];
/* check generation, collision bit, and type bit */
if ((entry->ie_bits & (IE_BITS_GEN_MASK|
IE_BITS_COLLISION|
MACH_PORT_TYPE_SEND_ONCE)) !=
(gen | MACH_PORT_TYPE_SEND_ONCE))
goto abort_reply_dest_copyin;
/* optimized ipc_right_copyin */
assert(IE_BITS_TYPE(entry->ie_bits) ==
MACH_PORT_TYPE_SEND_ONCE);
assert(IE_BITS_UREFS(entry->ie_bits) == 1);
assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0);
if (entry->ie_request != 0)
goto abort_reply_dest_copyin;
dest_port = (ipc_port_t) entry->ie_object;
assert(dest_port != IP_NULL);
ip_lock(dest_port);
if (!ip_active(dest_port)) {
ip_unlock(dest_port);
goto abort_reply_dest_copyin;
}
assert(dest_port->ip_sorights > 0);
/* optimized ipc_entry_dealloc */
entry->ie_next = table->ie_next;
table->ie_next = index;
entry->ie_bits = gen;
entry->ie_object = IO_NULL;
}
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
0);
kmsg->ikm_header.msgh_remote_port =
(mach_port_t) dest_port;
/* make sure we can queue to the destination */
assert(dest_port->ip_receiver != ipc_space_kernel);
/* optimized ipc_entry_lookup/ipc_mqueue_copyin */
{
ipc_entry_t entry;
ipc_entry_bits_t bits;
{
mach_port_index_t index;
mach_port_gen_t gen;
index = MACH_PORT_INDEX(rcv_name);
gen = MACH_PORT_GEN(rcv_name);
if (index >= size)
goto abort_reply_rcv_copyin;
entry = &table[index];
bits = entry->ie_bits;
/* check generation number */
if ((bits & IE_BITS_GEN_MASK) != gen)
goto abort_reply_rcv_copyin;
}
/* check type bits; looking for receive or set */
if (bits & MACH_PORT_TYPE_PORT_SET) {
ipc_pset_t rcv_pset;
rcv_pset = (ipc_pset_t) entry->ie_object;
assert(rcv_pset != IPS_NULL);
ips_lock(rcv_pset);
assert(ips_active(rcv_pset));
rcv_object = (ipc_object_t) rcv_pset;
rcv_mqueue = &rcv_pset->ips_messages;
} else if (bits & MACH_PORT_TYPE_RECEIVE) {
ipc_port_t rcv_port;
rcv_port = (ipc_port_t) entry->ie_object;
assert(rcv_port != IP_NULL);
if (!ip_lock_try(rcv_port))
goto abort_reply_rcv_copyin;
assert(ip_active(rcv_port));
if (rcv_port->ip_pset != IPS_NULL) {
ip_unlock(rcv_port);
goto abort_reply_rcv_copyin;
}
rcv_object = (ipc_object_t) rcv_port;
rcv_mqueue = &rcv_port->ip_messages;
} else
goto abort_reply_rcv_copyin;
}
is_write_unlock(space);
io_reference(rcv_object);
imq_lock(rcv_mqueue);
io_unlock(rcv_object);
goto fast_send_receive;
abort_reply_dest_copyin:
is_write_unlock(space);
goto slow_copyin;
abort_reply_rcv_copyin:
ip_unlock(dest_port);
is_write_unlock(space);
goto slow_send;
}
default:
goto slow_copyin;
}
/*NOTREACHED*/
fast_send_receive:
/*
* optimized ipc_mqueue_send/ipc_mqueue_receive
*
* Finished get/copyin of kmsg and copyin of rcv_name.
* space is unlocked, dest_port is locked,
* we can queue kmsg to dest_port,
* rcv_mqueue is locked, rcv_object holds a ref,
* if rcv_object is a port it isn't in a port set
*
* Note that if simple locking is turned off,
* then we could have dest_mqueue == rcv_mqueue
* and not abort when we try to lock dest_mqueue.
*/
assert(ip_active(dest_port));
assert(dest_port->ip_receiver != ipc_space_kernel);
assert((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
(MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
MACH_MSG_TYPE_PORT_SEND_ONCE));
assert((kmsg->ikm_header.msgh_bits &
MACH_MSGH_BITS_CIRCULAR) == 0);
{
ipc_mqueue_t dest_mqueue;
ipc_thread_t receiver;
{
ipc_pset_t dest_pset;
dest_pset = dest_port->ip_pset;
if (dest_pset == IPS_NULL)
dest_mqueue = &dest_port->ip_messages;
else
dest_mqueue = &dest_pset->ips_messages;
}
if (!imq_lock_try(dest_mqueue)) {
abort_send_receive:
ip_unlock(dest_port);
imq_unlock(rcv_mqueue);
ipc_object_release(rcv_object);
goto slow_send;
}
receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
if ((receiver == ITH_NULL) ||
(ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
!= IKM_NULL)) {
imq_unlock(dest_mqueue);
goto abort_send_receive;
}
/*
* There is a receiver thread waiting, and
* there is no reply message for us to pick up.
* We have hope of hand-off, so save state.
*/
self->ith_msg = msg;
self->ith_rcv_size = rcv_size;
self->ith_object = rcv_object;
self->ith_mqueue = rcv_mqueue;
if ((receiver->swap_func == (void (*)()) mach_msg_continue) &&
thread_handoff(self, mach_msg_continue, receiver)) {
assert(current_thread() == receiver);
/*
* We can use the optimized receive code,
* because the receiver is using no options.
*/
} else if ((receiver->swap_func ==
(void (*)()) exception_raise_continue) &&
thread_handoff(self, mach_msg_continue, receiver)) {
counter(c_mach_msg_trap_block_exc++);
assert(current_thread() == receiver);
/*
* We are a reply message coming back through
* the optimized exception-handling path.
* Finish with rcv_mqueue and dest_mqueue,
* and then jump to exception code with
* dest_port still locked. We don't bother
* with a sequence number in this case.
*/
ipc_thread_enqueue_macro(
&rcv_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(rcv_mqueue);
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
imq_unlock(dest_mqueue);
exception_raise_continue_fast(dest_port, kmsg);
/*NOTREACHED*/
return MACH_MSG_SUCCESS;
} else if ((send_size <= receiver->ith_msize) &&
thread_handoff(self, mach_msg_continue, receiver)) {
assert(current_thread() == receiver);
if ((receiver->swap_func ==
(void (*)()) mach_msg_receive_continue) &&
((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) {
/*
* We can still use the optimized code.
*/
} else {
counter(c_mach_msg_trap_block_slow++);
/*
* We are running as the receiver,
* but we can't use the optimized code.
* Finish send/receive processing.
*/
dest_port->ip_msgcount++;
ip_unlock(dest_port);
ipc_thread_enqueue_macro(
&rcv_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(rcv_mqueue);
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
receiver->ith_state = MACH_MSG_SUCCESS;
receiver->ith_kmsg = kmsg;
receiver->ith_seqno = dest_port->ip_seqno++;
imq_unlock(dest_mqueue);
/*
* Call the receiver's continuation.
*/
receiver->wait_result = THREAD_AWAKENED;
(*receiver->swap_func)();
/*NOTREACHED*/
return MACH_MSG_SUCCESS;
}
} else {
/*
* The receiver can't accept the message,
* or we can't switch to the receiver.
*/
imq_unlock(dest_mqueue);
goto abort_send_receive;
}
counter(c_mach_msg_trap_block_fast++);
/*
* Safe to unlock dest_port now that we are
* committed to this path, because we hold
* dest_mqueue locked. We never bother changing
* dest_port->ip_msgcount.
*/
ip_unlock(dest_port);
/*
* We need to finish preparing self for its
* time asleep in rcv_mqueue.
*/
ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(rcv_mqueue);
/*
* Finish extracting receiver from dest_mqueue.
*/
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
imq_unlock(dest_mqueue);
/*
* We don't have to do any post-dequeue processing of
* the message. We never incremented ip_msgcount, we
* know it has no msg-accepted request, and blocked
* senders aren't a worry because we found the port
* with a receiver waiting.
*/
self = receiver;
space = self->task->itk_space;
msg = self->ith_msg;
rcv_size = self->ith_rcv_size;
rcv_object = self->ith_object;
/* inline ipc_object_release */
io_lock(rcv_object);
io_release(rcv_object);
io_check_unlock(rcv_object);
}
fast_copyout:
/*
* Nothing locked and no references held, except
* we have kmsg with msgh_seqno filled in. Must
* still check against rcv_size and do
* ipc_kmsg_copyout/ipc_kmsg_put.
*/
assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port
== dest_port);
reply_size = kmsg->ikm_header.msgh_size;
if (rcv_size < reply_size)
goto slow_copyout;
/* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
switch (kmsg->ikm_header.msgh_bits) {
case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE): {
ipc_port_t reply_port =
(ipc_port_t) kmsg->ikm_header.msgh_local_port;
mach_port_t dest_name, reply_name;
/* receiving a request message */
if (!IP_VALID(reply_port))
goto slow_copyout;
is_write_lock(space);
assert(space->is_active);
/*
* To do an atomic copyout, need simultaneous
* locks on both ports and the space. If
* dest_port == reply_port, and simple locking is
* enabled, then we will abort. Otherwise it's
* OK to unlock twice.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port))
goto abort_request_copyout;
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
goto abort_request_copyout;
}
assert(reply_port->ip_sorights > 0);
ip_unlock(reply_port);
{
ipc_entry_t table;
ipc_entry_t entry;
mach_port_index_t index;
/* optimized ipc_entry_get */
table = space->is_table;
index = table->ie_next;
if (index == 0)
goto abort_request_copyout;
entry = &table[index];
table->ie_next = entry->ie_next;
entry->ie_request = 0;
{
mach_port_gen_t gen;
assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
gen = entry->ie_bits + IE_BITS_GEN_ONE;
reply_name = MACH_PORT_MAKE(index, gen);
/* optimized ipc_right_copyout */
entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
}
assert(MACH_PORT_VALID(reply_name));
entry->ie_object = (ipc_object_t) reply_port;
is_write_unlock(space);
}
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_srights > 0);
ip_release(dest_port);
if (dest_port->ip_receiver == space)
dest_name = dest_port->ip_receiver_name;
else
dest_name = MACH_PORT_NULL;
if ((--dest_port->ip_srights == 0) &&
(dest_port->ip_nsrequest != IP_NULL)) {
ipc_port_t nsrequest;
mach_port_mscount_t mscount;
/* a rather rare case */
nsrequest = dest_port->ip_nsrequest;
mscount = dest_port->ip_mscount;
dest_port->ip_nsrequest = IP_NULL;
ip_unlock(dest_port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(dest_port);
if (! ipc_port_flag_protected_payload(dest_port)) {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PORT_SEND);
kmsg->ikm_header.msgh_local_port = dest_name;
} else {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PROTECTED_PAYLOAD);
kmsg->ikm_header.msgh_protected_payload =
dest_port->ip_protected_payload;
}
kmsg->ikm_header.msgh_remote_port = reply_name;
goto fast_put;
abort_request_copyout:
ip_unlock(dest_port);
is_write_unlock(space);
goto slow_copyout;
}
case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
mach_port_t dest_name;
/* receiving a reply message */
ip_lock(dest_port);
if (!ip_active(dest_port))
goto slow_copyout;
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_sorights > 0);
if (dest_port->ip_receiver == space) {
ip_release(dest_port);
dest_port->ip_sorights--;
dest_name = dest_port->ip_receiver_name;
ip_unlock(dest_port);
} else {
ip_unlock(dest_port);
ipc_notify_send_once(dest_port);
dest_name = MACH_PORT_NULL;
}
if (! ipc_port_flag_protected_payload(dest_port)) {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PORT_SEND_ONCE);
kmsg->ikm_header.msgh_local_port = dest_name;
} else {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PROTECTED_PAYLOAD);
kmsg->ikm_header.msgh_protected_payload =
dest_port->ip_protected_payload;
}
kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
goto fast_put;
}
case MACH_MSGH_BITS_COMPLEX|
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
mach_port_t dest_name;
/* receiving a complex reply message */
ip_lock(dest_port);
if (!ip_active(dest_port))
goto slow_copyout;
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_sorights > 0);
if (dest_port->ip_receiver == space) {
ip_release(dest_port);
dest_port->ip_sorights--;
dest_name = dest_port->ip_receiver_name;
ip_unlock(dest_port);
} else {
ip_unlock(dest_port);
ipc_notify_send_once(dest_port);
dest_name = MACH_PORT_NULL;
}
if (! ipc_port_flag_protected_payload(dest_port)) {
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS_COMPLEX
| MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PORT_SEND_ONCE);
kmsg->ikm_header.msgh_local_port = dest_name;
} else {
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS_COMPLEX
| MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PROTECTED_PAYLOAD);
kmsg->ikm_header.msgh_protected_payload =
dest_port->ip_protected_payload;
}
kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
mr = ipc_kmsg_copyout_body(
(vm_offset_t) (&kmsg->ikm_header + 1),
(vm_offset_t) &kmsg->ikm_header
+ kmsg->ikm_header.msgh_size,
space,
current_map());
if (mr != MACH_MSG_SUCCESS) {
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
return mr | MACH_RCV_BODY_ERROR;
}
goto fast_put;
}
default:
goto slow_copyout;
}
/*NOTREACHED*/
fast_put:
/*
* We have the reply message data in kmsg,
* and the reply message size in reply_size.
* Just need to copy it out to the user and free kmsg.
* We must check ikm_cache after copyoutmsg.
*/
ikm_check_initialized(kmsg, kmsg->ikm_size);
if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
copyoutmsg(&kmsg->ikm_header, msg,
reply_size) ||
(ikm_cache() != IKM_NULL))
goto slow_put;
ikm_cache() = kmsg;
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
return MACH_MSG_SUCCESS; /* help for the compiler */
/*
* The slow path has a few non-register temporary
* variables used only for call-by-reference.
*/
{
ipc_kmsg_t temp_kmsg;
mach_port_seqno_t temp_seqno;
ipc_object_t temp_rcv_object;
ipc_mqueue_t temp_rcv_mqueue;
slow_get:
/*
* No locks, references, or messages held.
* Still have to get the request, send it,
* receive reply, etc.
*/
mr = ipc_kmsg_get(msg, send_size, &temp_kmsg);
if (mr != MACH_MSG_SUCCESS) {
thread_syscall_return(mr);
/*NOTREACHED*/
}
kmsg = temp_kmsg;
/* try to get back on optimized path */
goto fast_copyin;
slow_copyin:
/*
* We have the message data in kmsg, but
* we still need to copyin, send it,
* receive a reply, and do copyout.
*/
mr = ipc_kmsg_copyin(kmsg, space, current_map(),
MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ikm_free(kmsg);
thread_syscall_return(mr);
/*NOTREACHED*/
}
/* try to get back on optimized path */
if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR)
goto slow_send;
dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
assert(IP_VALID(dest_port));
ip_lock(dest_port);
if (dest_port->ip_receiver == ipc_space_kernel) {
assert(ip_active(dest_port));
ip_unlock(dest_port);
goto kernel_send;
}
if (ip_active(dest_port) &&
((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
(MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
MACH_MSG_TYPE_PORT_SEND_ONCE)))
{
/*
* Try an optimized ipc_mqueue_copyin.
* It will work if this is a request message.
*/
ipc_port_t reply_port;
reply_port = (ipc_port_t)
kmsg->ikm_header.msgh_local_port;
if (IP_VALID(reply_port)) {
if (ip_lock_try(reply_port)) {
if (ip_active(reply_port) &&
reply_port->ip_receiver == space &&
reply_port->ip_receiver_name == rcv_name &&
reply_port->ip_pset == IPS_NULL)
{
/* Grab a reference to the reply port. */
rcv_object = (ipc_object_t) reply_port;
io_reference(rcv_object);
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
io_unlock(rcv_object);
goto fast_send_receive;
}
ip_unlock(reply_port);
}
}
}
ip_unlock(dest_port);
goto slow_send;
kernel_send:
/*
* Special case: send message to kernel services.
* The request message has been copied into the
* kmsg. Nothing is locked.
*/
{
ipc_port_t reply_port;
/*
* Perform the kernel function.
*/
kmsg = ipc_kobject_server(kmsg);
if (kmsg == IKM_NULL) {
/*
* No reply. Take the
* slow receive path.
*/
goto slow_get_rcv_port;
}
/*
* Check that:
* the reply port is alive
* we hold the receive right
* the name has not changed.
* the port is not in a set
* If any of these are not true,
* we cannot directly receive the reply
* message.
*/
reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
ip_lock(reply_port);
if ((!ip_active(reply_port)) ||
(reply_port->ip_receiver != space) ||
(reply_port->ip_receiver_name != rcv_name) ||
(reply_port->ip_pset != IPS_NULL))
{
ip_unlock(reply_port);
ipc_mqueue_send_always(kmsg);
goto slow_get_rcv_port;
}
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
/* keep port locked, and don`t change ref count yet */
/*
* If there are messages on the port
* or other threads waiting for a message,
* we cannot directly receive the reply.
*/
if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads)
!= ITH_NULL) ||
(ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
!= IKM_NULL))
{
imq_unlock(rcv_mqueue);
ip_unlock(reply_port);
ipc_mqueue_send_always(kmsg);
goto slow_get_rcv_port;
}
/*
* We can directly receive this reply.
* Since the kernel reply never blocks,
* it holds no message_accepted request.
* Since there were no messages queued
* on the reply port, there should be
* no threads blocked waiting to send.
*/
assert(kmsg->ikm_marequest == IMAR_NULL);
assert(ipc_thread_queue_first(&reply_port->ip_blocked)
== ITH_NULL);
dest_port = reply_port;
kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
imq_unlock(rcv_mqueue);
/*
* inline ipc_object_release.
* Port is still locked.
* Reference count was not incremented.
*/
ip_check_unlock(reply_port);
/* copy out the kernel reply */
goto fast_copyout;
}
slow_send:
/*
* Nothing is locked. We have acquired kmsg, but
* we still need to send it and receive a reply.
*/
mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
current_map());
assert(kmsg->ikm_marequest == IMAR_NULL);
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
slow_get_rcv_port:
/*
* We have sent the message. Copy in the receive port.
*/
mr = ipc_mqueue_copyin(space, rcv_name,
&temp_rcv_mqueue, &temp_rcv_object);
if (mr != MACH_MSG_SUCCESS) {
thread_syscall_return(mr);
/*NOTREACHED*/
}
rcv_mqueue = temp_rcv_mqueue;
rcv_object = temp_rcv_object;
/* hold ref for rcv_object; rcv_mqueue is locked */
/*
slow_receive:
*/
/*
* Now we have sent the request and copied in rcv_name,
* so rcv_mqueue is locked and hold ref for rcv_object.
* Just receive a reply and try to get back to fast path.
*
* ipc_mqueue_receive may not return, because if we block
* then our kernel stack may be discarded. So we save
* state here for mach_msg_continue to pick up.
*/
self->ith_msg = msg;
self->ith_rcv_size = rcv_size;
self->ith_object = rcv_object;
self->ith_mqueue = rcv_mqueue;
mr = ipc_mqueue_receive(rcv_mqueue,
MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, mach_msg_continue,
&temp_kmsg, &temp_seqno);
/* rcv_mqueue is unlocked */
ipc_object_release(rcv_object);
if (mr != MACH_MSG_SUCCESS) {
thread_syscall_return(mr);
/*NOTREACHED*/
}
(kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno;
dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
goto fast_copyout;
slow_copyout:
/*
* Nothing locked and no references held, except
* we have kmsg with msgh_seqno filled in. Must
* still check against rcv_size and do
* ipc_kmsg_copyout/ipc_kmsg_put.
*/
reply_size = kmsg->ikm_header.msgh_size;
if (rcv_size < reply_size) {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
thread_syscall_return(MACH_RCV_TOO_LARGE);
/*NOTREACHED*/
}
mr = ipc_kmsg_copyout(kmsg, space, current_map(),
MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
} else {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
}
thread_syscall_return(mr);
/*NOTREACHED*/
}
/* try to get back on optimized path */
goto fast_put;
slow_put:
mr = ipc_kmsg_put(msg, kmsg, reply_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
} else if (option == MACH_SEND_MSG) {
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_kmsg_t kmsg;
mr = ipc_kmsg_get(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ikm_free(kmsg);
return mr;
}
mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
assert(kmsg->ikm_marequest == IMAR_NULL);
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
}
return mr;
} else if (option == MACH_RCV_MSG) {
ipc_thread_t self = current_thread();
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_object_t object;
ipc_mqueue_t mqueue;
ipc_kmsg_t kmsg;
mach_port_seqno_t seqno;
mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
if (mr != MACH_MSG_SUCCESS)
return mr;
/* hold ref for object; mqueue is locked */
/*
* ipc_mqueue_receive may not return, because if we block
* then our kernel stack may be discarded. So we save
* state here for mach_msg_continue to pick up.
*/
self->ith_msg = msg;
self->ith_rcv_size = rcv_size;
self->ith_object = object;
self->ith_mqueue = mqueue;
mr = ipc_mqueue_receive(mqueue,
MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, mach_msg_continue,
&kmsg, &seqno);
/* mqueue is unlocked */
ipc_object_release(object);
if (mr != MACH_MSG_SUCCESS)
return mr;
kmsg->ikm_header.msgh_seqno = seqno;
if (rcv_size < kmsg->ikm_header.msgh_size) {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
return MACH_RCV_TOO_LARGE;
}
mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
} else {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
}
return mr;
}
return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
} else if (option == MACH_MSG_OPTION_NONE) {
/*
* We can measure the "null mach_msg_trap"
* (syscall entry and thread_syscall_return exit)
* with this path.
*/
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
}
if (option & MACH_SEND_MSG) {
mr = mach_msg_send(msg, option, send_size,
time_out, notify);
if (mr != MACH_MSG_SUCCESS)
return mr;
}
if (option & MACH_RCV_MSG) {
mr = mach_msg_receive(msg, option, rcv_size, rcv_name,
time_out, notify);
if (mr != MACH_MSG_SUCCESS)
return mr;
}
return MACH_MSG_SUCCESS;
}
wait_result_t
ipc_mqueue_receive_on_thread(
ipc_mqueue_t mqueue,
mach_msg_option_t option,
mach_msg_size_t max_size,
mach_msg_timeout_t rcv_timeout,
int interruptible,
thread_t thread)
{
ipc_kmsg_queue_t kmsgs;
wait_result_t wresult;
uint64_t deadline;
spl_t s;
#if CONFIG_MACF_MACH
ipc_labelh_t lh;
task_t task;
int rc;
#endif
s = splsched();
imq_lock(mqueue);
if (imq_is_set(mqueue)) {
queue_t q;
q = &mqueue->imq_preposts;
/*
* If we are waiting on a portset mqueue, we need to see if
* any of the member ports have work for us. Ports that
* have (or recently had) messages will be linked in the
* prepost queue for the portset. By holding the portset's
* mqueue lock during the search, we tie up any attempts by
* mqueue_deliver or portset membership changes that may
* cross our path.
*/
search_set:
while(!queue_empty(q)) {
wait_queue_link_t wql;
ipc_mqueue_t port_mq;
queue_remove_first(q, wql, wait_queue_link_t, wql_preposts);
assert(!wql_is_preposted(wql));
/*
* This is a lock order violation, so we have to do it
* "softly," putting the link back on the prepost list
* if it fails (at the tail is fine since the order of
* handling messages from different sources in a set is
* not guaranteed and we'd like to skip to the next source
* if one is available).
*/
port_mq = (ipc_mqueue_t)wql->wql_queue;
if (!imq_lock_try(port_mq)) {
queue_enter(q, wql, wait_queue_link_t, wql_preposts);
imq_unlock(mqueue);
splx(s);
mutex_pause(0);
s = splsched();
imq_lock(mqueue);
goto search_set; /* start again at beginning - SMP */
}
/*
* If there are no messages on this queue, just skip it
* (we already removed the link from the set's prepost queue).
*/
kmsgs = &port_mq->imq_messages;
if (ipc_kmsg_queue_first(kmsgs) == IKM_NULL) {
imq_unlock(port_mq);
continue;
}
/*
* There are messages, so reinsert the link back
* at the tail of the preposted queue (for fairness)
* while we still have the portset mqueue locked.
*/
queue_enter(q, wql, wait_queue_link_t, wql_preposts);
imq_unlock(mqueue);
/*
* Continue on to handling the message with just
* the port mqueue locked.
*/
ipc_mqueue_select_on_thread(port_mq, option, max_size, thread);
imq_unlock(port_mq);
#if CONFIG_MACF_MACH
if (thread->task != TASK_NULL &&
thread->ith_kmsg != NULL &&
thread->ith_kmsg->ikm_sender != NULL) {
lh = thread->ith_kmsg->ikm_sender->label;
tasklabel_lock(thread->task);
ip_lock(lh->lh_port);
rc = mac_port_check_receive(&thread->task->maclabel,
&lh->lh_label);
ip_unlock(lh->lh_port);
tasklabel_unlock(thread->task);
if (rc)
thread->ith_state = MACH_RCV_INVALID_DATA;
}
#endif
splx(s);
return THREAD_NOT_WAITING;
}
} else {
/*
* Receive on a single port. Just try to get the messages.
*/
kmsgs = &mqueue->imq_messages;
if (ipc_kmsg_queue_first(kmsgs) != IKM_NULL) {
ipc_mqueue_select_on_thread(mqueue, option, max_size, thread);
imq_unlock(mqueue);
#if CONFIG_MACF_MACH
if (thread->task != TASK_NULL &&
thread->ith_kmsg != NULL &&
thread->ith_kmsg->ikm_sender != NULL) {
lh = thread->ith_kmsg->ikm_sender->label;
tasklabel_lock(thread->task);
ip_lock(lh->lh_port);
rc = mac_port_check_receive(&thread->task->maclabel,
&lh->lh_label);
ip_unlock(lh->lh_port);
tasklabel_unlock(thread->task);
if (rc)
thread->ith_state = MACH_RCV_INVALID_DATA;
}
#endif
splx(s);
return THREAD_NOT_WAITING;
}
}
/*
* Looks like we'll have to block. The mqueue we will
* block on (whether the set's or the local port's) is
* still locked.
*/
if (option & MACH_RCV_TIMEOUT) {
if (rcv_timeout == 0) {
imq_unlock(mqueue);
splx(s);
thread->ith_state = MACH_RCV_TIMED_OUT;
return THREAD_NOT_WAITING;
}
}
thread_lock(thread);
thread->ith_state = MACH_RCV_IN_PROGRESS;
thread->ith_option = option;
thread->ith_msize = max_size;
if (option & MACH_RCV_TIMEOUT)
clock_interval_to_deadline(rcv_timeout, 1000*NSEC_PER_USEC, &deadline);
else
deadline = 0;
wresult = wait_queue_assert_wait64_locked(&mqueue->imq_wait_queue,
IPC_MQUEUE_RECEIVE,
interruptible, deadline,
thread);
/* preposts should be detected above, not here */
if (wresult == THREAD_AWAKENED)
panic("ipc_mqueue_receive_on_thread: sleep walking");
thread_unlock(thread);
imq_unlock(mqueue);
splx(s);
return wresult;
}
