kern_return_t
ipc_object_alloc_dead_name(
ipc_space_t space,
mach_port_t name)
{
ipc_entry_t entry;
kern_return_t kr;
is_write_lock(space);
kr = ipc_entry_alloc_name(space, name, &entry);
if (kr != KERN_SUCCESS) {
is_write_unlock(space);
return kr;
}
if (ipc_right_inuse(space, name, entry))
return KERN_NAME_EXISTS;
/* null object, MACH_PORT_TYPE_DEAD_NAME, 1 uref */
assert(entry->ie_object == IO_NULL);
entry->ie_bits |= MACH_PORT_TYPE_DEAD_NAME | 1;
is_write_unlock(space);
return KERN_SUCCESS;
}
kern_return_t
ipc_right_lookup_write(
ipc_space_t space,
mach_port_t name,
ipc_entry_t *entryp)
{
ipc_entry_t entry;
assert(space != IS_NULL);
is_write_lock(space);
if (!space->is_active) {
is_write_unlock(space);
return KERN_INVALID_TASK;
}
if ((entry = ipc_entry_lookup(space, name)) == IE_NULL) {
is_write_unlock(space);
return KERN_INVALID_NAME;
}
*entryp = entry;
return KERN_SUCCESS;
}
void
ipc_space_terminate(
ipc_space_t space)
{
ipc_entry_t table;
ipc_entry_num_t size;
mach_port_index_t index;
assert(space != IS_NULL);
is_write_lock(space);
if (!is_active(space)) {
is_write_unlock(space);
return;
}
is_mark_inactive(space);
/*
* If somebody is trying to grow the table,
* we must wait until they finish and figure
* out the space died.
*/
while (is_growing(space))
is_write_sleep(space);
is_write_unlock(space);
/*
* Now we can futz with it unlocked.
*/
table = space->is_table;
size = space->is_table_size;
for (index = 0; index < size; index++) {
ipc_entry_t entry = &table[index];
mach_port_type_t type;
type = IE_BITS_TYPE(entry->ie_bits);
if (type != MACH_PORT_TYPE_NONE) {
mach_port_name_t name;
name = MACH_PORT_MAKE(index,
IE_BITS_GEN(entry->ie_bits));
ipc_right_terminate(space, name, entry);
}
}
it_entries_free(space->is_table_next-1, table);
space->is_table_size = 0;
/*
* Because the space is now dead,
* we must release the "active" reference for it.
* Our caller still has his reference.
*/
is_release(space);
}
kern_return_t
ipc_object_alloc_name(
ipc_space_t space,
ipc_object_type_t otype,
mach_port_type_t type,
mach_port_urefs_t urefs,
mach_port_t name,
ipc_object_t *objectp)
{
ipc_object_t object;
ipc_entry_t entry;
kern_return_t kr;
assert(otype < IOT_NUMBER);
assert((type & MACH_PORT_TYPE_ALL_RIGHTS) == type);
assert(type != MACH_PORT_TYPE_NONE);
assert(urefs <= MACH_PORT_UREFS_MAX);
object = io_alloc(otype);
if (object == IO_NULL)
return KERN_RESOURCE_SHORTAGE;
if (otype == IOT_PORT) {
ipc_port_t port = (ipc_port_t)object;
memset(port, 0, sizeof(*port));
} else if (otype == IOT_PORT_SET) {
ipc_pset_t pset = (ipc_pset_t)object;
memset(pset, 0, sizeof(*pset));
}
is_write_lock(space);
kr = ipc_entry_alloc_name(space, name, &entry);
if (kr != KERN_SUCCESS) {
is_write_unlock(space);
io_free(otype, object);
return kr;
}
if (ipc_right_inuse(space, name, entry)) {
io_free(otype, object);
return KERN_NAME_EXISTS;
}
entry->ie_bits |= type | urefs;
entry->ie_object = object;
io_lock_init(object);
io_lock(object);
is_write_unlock(space);
object->io_references = 1; /* for entry, not caller */
object->io_bits = io_makebits(TRUE, otype, 0);
*objectp = object;
return KERN_SUCCESS;
}
/*
* ipc_space_clean - remove all port references from an ipc space.
*
* In order to follow the traditional semantic, ipc_space_destroy
* will not destroy the entire port table of a shared space. Instead
* it will simply clear its own sub-space.
*/
void
ipc_space_clean(
ipc_space_t space)
{
ipc_entry_t table;
ipc_entry_num_t size;
mach_port_index_t index;
/*
* If somebody is trying to grow the table,
* we must wait until they finish and figure
* out the space died.
*/
retry:
is_write_lock(space);
while (is_growing(space))
is_write_sleep(space);
if (!is_active(space)) {
is_write_unlock(space);
return;
}
/*
* Now we can futz with it since we have the write lock.
*/
table = space->is_table;
size = space->is_table_size;
for (index = 0; index < size; index++) {
ipc_entry_t entry = &table[index];
mach_port_type_t type;
type = IE_BITS_TYPE(entry->ie_bits);
if (type != MACH_PORT_TYPE_NONE) {
mach_port_name_t name = MACH_PORT_MAKE(index,
IE_BITS_GEN(entry->ie_bits));
ipc_right_destroy(space, name, entry, FALSE, 0); /* unlocks space */
goto retry;
}
}
/*
* JMM - Now the table is cleaned out. We don't bother shrinking the
* size of the table at this point, but we probably should if it is
* really large.
*/
is_write_unlock(space);
}
kern_return_t
mach_set_port_label(
ipc_space_t space,
mach_port_name_t name,
vm_offset_t labelstr)
{
#ifdef MAC
ipc_entry_t entry;
kern_return_t kr;
struct label inl;
int rc;
if (space == IS_NULL || space->is_task == NULL)
return KERN_INVALID_TASK;
if (!MACH_PORT_VALID(name))
return KERN_INVALID_NAME;
mac_init_port_label(&inl);
rc = mac_internalize_port_label(&inl, labelstr);
if (rc)
return KERN_INVALID_ARGUMENT;
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
if (io_otype(entry->ie_object) != IOT_PORT) {
is_write_unlock(space);
return KERN_INVALID_RIGHT;
}
ipc_port_t port = (ipc_port_t) entry->ie_object;
ip_lock(port);
rc = mac_check_port_relabel(&space->is_task->maclabel,
&port->ip_label, &inl);
if (rc)
kr = KERN_NO_ACCESS;
else
mac_copy_port_label(&inl, &port->ip_label);
ip_unlock(port);
is_write_unlock(space);
return kr;
#else
return KERN_INVALID_ARGUMENT;
#endif
}
kern_return_t
mach_port_type(
ipc_space_t space,
mach_port_name_t name,
mach_port_type_t *typep)
{
mach_port_urefs_t urefs;
ipc_entry_t entry;
kern_return_t kr;
if (space == IS_NULL)
return KERN_INVALID_TASK;
if (name == MACH_PORT_NULL)
return KERN_INVALID_NAME;
if (name == MACH_PORT_DEAD) {
*typep = MACH_PORT_TYPE_DEAD_NAME;
return KERN_SUCCESS;
}
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
kr = ipc_right_info(space, name, entry, typep, &urefs);
if (kr == KERN_SUCCESS)
is_write_unlock(space);
/* space is unlocked */
return kr;
}
kern_return_t
ipc_pset_alloc(
ipc_space_t space,
mach_port_name_t *namep,
ipc_pset_t *psetp)
{
ipc_pset_t pset;
mach_port_name_t name;
kern_return_t kr;
kr = ipc_object_alloc(space, IOT_PORT_SET,
MACH_PORT_TYPE_PORT_SET, 0,
&name, (ipc_object_t *) &pset);
if (kr != KERN_SUCCESS)
return kr;
/* pset and space are locked */
pset->ips_local_name = name;
ipc_mqueue_init(&pset->ips_messages, TRUE /* set */);
is_write_unlock(space);
*namep = name;
*psetp = pset;
return KERN_SUCCESS;
}
kern_return_t
ipc_pset_alloc(
ipc_space_t space,
mach_port_name_t *namep,
ipc_pset_t *psetp)
{
ipc_pset_t pset;
mach_port_name_t name;
kern_return_t kr;
uint64_t reserved_link;
reserved_link = waitq_link_reserve(NULL);
kr = ipc_object_alloc(space, IOT_PORT_SET,
MACH_PORT_TYPE_PORT_SET, 0,
&name, (ipc_object_t *) &pset);
if (kr != KERN_SUCCESS) {
waitq_link_release(reserved_link);
return kr;
}
/* pset and space are locked */
pset->ips_local_name = name;
ipc_mqueue_init(&pset->ips_messages, TRUE /* set */, &reserved_link);
is_write_unlock(space);
waitq_link_release(reserved_link);
*namep = name;
*psetp = pset;
return KERN_SUCCESS;
}
kern_return_t
ipc_object_copyin(
ipc_space_t space,
mach_port_name_t name,
mach_msg_type_name_t msgt_name,
ipc_object_t *objectp)
{
ipc_entry_t entry;
ipc_port_t soright;
kern_return_t kr;
/*
* Could first try a read lock when doing
* MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND,
* and MACH_MSG_TYPE_MAKE_SEND_ONCE.
*/
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
kr = ipc_right_copyin(space, name, entry,
msgt_name, TRUE,
objectp, &soright);
if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
if ((kr == KERN_SUCCESS) && (soright != IP_NULL))
ipc_notify_port_deleted(soright, name);
return kr;
}
kern_return_t
ipc_object_rename(
ipc_space_t space,
mach_port_name_t oname,
mach_port_name_t nname)
{
ipc_entry_t oentry, nentry;
kern_return_t kr;
kr = ipc_entry_alloc_name(space, nname, &nentry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
if (ipc_right_inuse(space, nname, nentry)) {
/* space is unlocked */
return KERN_NAME_EXISTS;
}
/* don't let ipc_entry_lookup see the uninitialized new entry */
if ((oname == nname) ||
((oentry = ipc_entry_lookup(space, oname)) == IE_NULL)) {
ipc_entry_dealloc(space, nname, nentry);
is_write_unlock(space);
return KERN_INVALID_NAME;
}
kr = ipc_right_rename(space, oname, oentry, nname, nentry);
/* space is unlocked */
return kr;
}
/*
* Get a label handle representing the given port's port label.
*/
kern_return_t
mach_get_label(
ipc_space_t space,
mach_port_name_t name,
mach_port_name_t *outlabel)
{
ipc_entry_t entry;
ipc_port_t port;
struct label outl;
kern_return_t kr;
int dead;
if (!MACH_PORT_VALID(name))
return KERN_INVALID_NAME;
/* Lookup the port name in the task's space. */
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
port = (ipc_port_t) entry->ie_object;
dead = ipc_right_check(space, port, name, entry);
if (dead) {
is_write_unlock(space);
return KERN_INVALID_RIGHT;
}
/* port is now locked */
is_write_unlock(space);
/* Make sure we are not dealing with a label handle. */
if (ip_kotype(port) == IKOT_LABELH) {
/* already is a label handle! */
ip_unlock(port);
return KERN_INVALID_ARGUMENT;
}
/* Copy the port label and stash it in a new label handle. */
mac_init_port_label(&outl);
mac_copy_port_label(&port->ip_label, &outl);
kr = labelh_new_user(space, &outl, outlabel);
ip_unlock(port);
return KERN_SUCCESS;
}
kern_return_t
ipc_object_alloc(
ipc_space_t space,
ipc_object_type_t otype,
mach_port_type_t type,
mach_port_urefs_t urefs,
mach_port_name_t *namep,
ipc_object_t *objectp)
{
ipc_object_t object;
ipc_entry_t entry;
kern_return_t kr;
assert(otype < IOT_NUMBER);
assert((type & MACH_PORT_TYPE_ALL_RIGHTS) == type);
assert(type != MACH_PORT_TYPE_NONE);
assert(urefs <= MACH_PORT_UREFS_MAX);
object = io_alloc(otype);
if (object == IO_NULL)
return KERN_RESOURCE_SHORTAGE;
if (otype == IOT_PORT) {
ipc_port_t port = (ipc_port_t)object;
bzero((char *)port, sizeof(*port));
#if CONFIG_MACF_MACH
mac_port_label_init(&port->ip_label);
#endif
} else if (otype == IOT_PORT_SET) {
ipc_pset_t pset = (ipc_pset_t)object;
bzero((char *)pset, sizeof(*pset));
}
io_lock_init(object);
*namep = CAST_MACH_PORT_TO_NAME(object);
kr = ipc_entry_alloc(space, namep, &entry);
if (kr != KERN_SUCCESS) {
io_free(otype, object);
return kr;
}
/* space is write-locked */
entry->ie_bits |= type | urefs;
entry->ie_object = object;
io_lock(object);
is_write_unlock(space);
object->io_references = 1; /* for entry, not caller */
object->io_bits = io_makebits(TRUE, otype, 0);
*objectp = object;
return KERN_SUCCESS;
}
kern_return_t
ipc_object_copyin(
ipc_space_t space,
mach_port_name_t name,
mach_msg_type_name_t msgt_name,
ipc_object_t *objectp)
{
ipc_entry_t entry;
ipc_port_t soright;
ipc_port_t release_port;
kern_return_t kr;
queue_head_t links_data;
queue_t links = &links_data;
wait_queue_link_t wql;
queue_init(links);
/*
* Could first try a read lock when doing
* MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND,
* and MACH_MSG_TYPE_MAKE_SEND_ONCE.
*/
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
release_port = IP_NULL;
kr = ipc_right_copyin(space, name, entry,
msgt_name, TRUE,
objectp, &soright,
&release_port,
links);
if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
while(!queue_empty(links)) {
wql = (wait_queue_link_t) dequeue(links);
wait_queue_link_free(wql);
}
if (release_port != IP_NULL)
ip_release(release_port);
if ((kr == KERN_SUCCESS) && (soright != IP_NULL))
ipc_notify_port_deleted(soright, name);
return kr;
}
kern_return_t
mach_get_label_text(
ipc_space_t space,
mach_port_name_t name,
vm_offset_t policies,
vm_offset_t outlabel)
{
ipc_entry_t entry;
kern_return_t kr;
struct label *l;
int dead;
if (space == IS_NULL || space->is_task == NULL)
return KERN_INVALID_TASK;
if (!MACH_PORT_VALID(name))
return KERN_INVALID_NAME;
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
dead = ipc_right_check(space, entry->ie_object, name, entry);
if (dead) {
is_write_unlock(space);
return KERN_INVALID_RIGHT;
}
/* object (port) is now locked */
is_write_unlock(space);
l = io_getlabel(entry->ie_object);
mac_externalize_port_label(l, policies, outlabel, 512, 0);
io_unlocklabel(entry->ie_object);
io_unlock(entry->ie_object);
return KERN_SUCCESS;
}
kern_return_t
ipc_object_alloc_dead(
ipc_space_t space,
mach_port_t *namep)
{
ipc_entry_t entry;
kern_return_t kr;
is_write_lock(space);
kr = ipc_entry_alloc(space, namep, &entry);
if (kr != KERN_SUCCESS) {
is_write_unlock(space);
return kr;
}
/* null object, MACH_PORT_TYPE_DEAD_NAME, 1 uref */
assert(entry->ie_object == IO_NULL);
entry->ie_bits |= MACH_PORT_TYPE_DEAD_NAME | 1;
is_write_unlock(space);
return KERN_SUCCESS;
}
boolean_t
ipc_right_inuse(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry)
{
ipc_entry_bits_t bits = entry->ie_bits;
if (IE_BITS_TYPE(bits) != MACH_PORT_TYPE_NONE) {
is_write_unlock(space);
return TRUE;
}
return FALSE;
}
kern_return_t
ipc_object_copyin(
ipc_space_t space,
mach_port_name_t name,
mach_msg_type_name_t msgt_name,
ipc_object_t *objectp)
{
ipc_entry_t entry;
ipc_port_t soright;
ipc_port_t release_port;
kern_return_t kr;
int assertcnt = 0;
/*
* Could first try a read lock when doing
* MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND,
* and MACH_MSG_TYPE_MAKE_SEND_ONCE.
*/
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
release_port = IP_NULL;
kr = ipc_right_copyin(space, name, entry,
msgt_name, TRUE,
objectp, &soright,
&release_port,
&assertcnt);
if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
#if IMPORTANCE_INHERITANCE
if (0 < assertcnt && ipc_importance_task_is_any_receiver_type(current_task()->task_imp_base)) {
ipc_importance_task_drop_internal_assertion(current_task()->task_imp_base, assertcnt);
}
#endif /* IMPORTANCE_INHERITANCE */
if (release_port != IP_NULL)
ip_release(release_port);
if ((kr == KERN_SUCCESS) && (soright != IP_NULL))
ipc_notify_port_deleted(soright, name);
return kr;
}
kern_return_t
ipc_port_alloc(
ipc_space_t space,
mach_port_name_t *namep,
ipc_port_t *portp)
{
ipc_port_t port;
mach_port_name_t name;
kern_return_t kr;
#if MACH_ASSERT
uintptr_t buf[IP_CALLSTACK_MAX];
ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);
#endif /* MACH_ASSERT */
kr = ipc_object_alloc(space, IOT_PORT,
MACH_PORT_TYPE_RECEIVE, 0,
&name, (ipc_object_t *) &port);
if (kr != KERN_SUCCESS)
return kr;
/* port and space are locked */
ipc_port_init(port, space, name);
#if MACH_ASSERT
ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);
#endif /* MACH_ASSERT */
/* unlock space after init */
is_write_unlock(space);
#if CONFIG_MACF_MACH
task_t issuer = current_task();
tasklabel_lock2 (issuer, space->is_task);
mac_port_label_associate(&issuer->maclabel, &space->is_task->maclabel,
&port->ip_label);
tasklabel_unlock2 (issuer, space->is_task);
#endif
*namep = name;
*portp = port;
return KERN_SUCCESS;
}
kern_return_t
mach_port_destroy(
ipc_space_t space,
mach_port_name_t name)
{
ipc_entry_t entry;
kern_return_t kr;
if (space == IS_NULL)
return KERN_INVALID_TASK;
if (!MACH_PORT_VALID(name))
return KERN_SUCCESS;
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
kr = ipc_right_destroy(space, name, entry);
is_write_unlock(space);
return kr;
}
mach_msg_return_t
mach_msg_trap(
mach_msg_header_t *msg,
mach_msg_option_t option,
mach_msg_size_t send_size,
mach_msg_size_t rcv_size,
mach_port_t rcv_name,
mach_msg_timeout_t time_out,
mach_port_t notify)
{
mach_msg_return_t mr;
/* first check for common cases */
if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) {
ipc_thread_t self = current_thread();
ipc_space_t space = self->task->itk_space;
ipc_kmsg_t kmsg;
ipc_port_t dest_port;
ipc_object_t rcv_object;
ipc_mqueue_t rcv_mqueue;
mach_msg_size_t reply_size;
/*
* This case is divided into ten sections, each
* with a label. There are five optimized
* sections and six unoptimized sections, which
* do the same thing but handle all possible
* cases and are slower.
*
* The five sections for an RPC are
* 1) Get request message into a buffer.
* (fast_get or slow_get)
* 2) Copyin request message and rcv_name.
* (fast_copyin or slow_copyin)
* 3) Enqueue request and dequeue reply.
* (fast_send_receive or
* slow_send and slow_receive)
* 4) Copyout reply message.
* (fast_copyout or slow_copyout)
* 5) Put reply message to user's buffer.
* (fast_put or slow_put)
*
* Keep the locking hierarchy firmly in mind.
* (First spaces, then ports, then port sets,
* then message queues.) Only a non-blocking
* attempt can be made to acquire locks out of
* order, or acquire two locks on the same level.
* Acquiring two locks on the same level will
* fail if the objects are really the same,
* unless simple locking is disabled. This is OK,
* because then the extra unlock does nothing.
*
* There are two major reasons these RPCs can't use
* ipc_thread_switch, and use slow_send/slow_receive:
* 1) Kernel RPCs.
* 2) Servers fall behind clients, so
* client doesn't find a blocked server thread and
* server finds waiting messages and can't block.
*/
/*
fast_get:
*/
/*
* optimized ipc_kmsg_get
*
* No locks, references, or messages held.
* We must clear ikm_cache before copyinmsg.
*/
if ((send_size > IKM_SAVED_MSG_SIZE) ||
(send_size < sizeof(mach_msg_header_t)) ||
(send_size & 3) ||
((kmsg = ikm_cache()) == IKM_NULL))
goto slow_get;
ikm_cache() = IKM_NULL;
ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
if (copyinmsg(msg, &kmsg->ikm_header,
send_size)) {
ikm_free(kmsg);
goto slow_get;
}
kmsg->ikm_header.msgh_size = send_size;
fast_copyin:
/*
* optimized ipc_kmsg_copyin/ipc_mqueue_copyin
*
* We have the request message data in kmsg.
* Must still do copyin, send, receive, etc.
*
* If the message isn't simple, we can't combine
* ipc_kmsg_copyin_header and ipc_mqueue_copyin,
* because copyin of the message body might
* affect rcv_name.
*/
switch (kmsg->ikm_header.msgh_bits) {
case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
MACH_MSG_TYPE_MAKE_SEND_ONCE): {
ipc_entry_t table;
ipc_entry_num_t size;
ipc_port_t reply_port;
/* sending a request message */
{
mach_port_index_t index;
mach_port_gen_t gen;
{
mach_port_t reply_name =
kmsg->ikm_header.msgh_local_port;
if (reply_name != rcv_name)
goto slow_copyin;
/* optimized ipc_entry_lookup of reply_name */
index = MACH_PORT_INDEX(reply_name);
gen = MACH_PORT_GEN(reply_name);
}
is_read_lock(space);
assert(space->is_active);
size = space->is_table_size;
table = space->is_table;
if (index >= size)
goto abort_request_copyin;
{
ipc_entry_t entry;
ipc_entry_bits_t bits;
entry = &table[index];
bits = entry->ie_bits;
/* check generation number and type bit */
if ((bits & (IE_BITS_GEN_MASK|
MACH_PORT_TYPE_RECEIVE)) !=
(gen | MACH_PORT_TYPE_RECEIVE))
goto abort_request_copyin;
reply_port = (ipc_port_t) entry->ie_object;
assert(reply_port != IP_NULL);
}
}
/* optimized ipc_entry_lookup of dest_name */
{
mach_port_index_t index;
mach_port_gen_t gen;
{
mach_port_t dest_name =
kmsg->ikm_header.msgh_remote_port;
index = MACH_PORT_INDEX(dest_name);
gen = MACH_PORT_GEN(dest_name);
}
if (index >= size)
goto abort_request_copyin;
{
ipc_entry_t entry;
ipc_entry_bits_t bits;
entry = &table[index];
bits = entry->ie_bits;
/* check generation number and type bit */
if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
(gen | MACH_PORT_TYPE_SEND))
goto abort_request_copyin;
assert(IE_BITS_UREFS(bits) > 0);
dest_port = (ipc_port_t) entry->ie_object;
assert(dest_port != IP_NULL);
}
}
/*
* To do an atomic copyin, need simultaneous
* locks on both ports and the space. If
* dest_port == reply_port, and simple locking is
* enabled, then we will abort. Otherwise it's
* OK to unlock twice.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
ip_unlock(dest_port);
goto abort_request_copyin;
}
is_read_unlock(space);
assert(dest_port->ip_srights > 0);
dest_port->ip_srights++;
ip_reference(dest_port);
assert(ip_active(reply_port));
assert(reply_port->ip_receiver_name ==
kmsg->ikm_header.msgh_local_port);
assert(reply_port->ip_receiver == space);
reply_port->ip_sorights++;
ip_reference(reply_port);
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE);
kmsg->ikm_header.msgh_remote_port =
(mach_port_t) dest_port;
kmsg->ikm_header.msgh_local_port =
(mach_port_t) reply_port;
/* make sure we can queue to the destination */
if (dest_port->ip_receiver == ipc_space_kernel) {
/*
* The kernel server has a reference to
* the reply port, which it hands back
* to us in the reply message. We do
* not need to keep another reference to
* it.
*/
ip_unlock(reply_port);
assert(ip_active(dest_port));
ip_unlock(dest_port);
goto kernel_send;
}
if (dest_port->ip_msgcount >= dest_port->ip_qlimit)
goto abort_request_send_receive;
/* optimized ipc_mqueue_copyin */
if (reply_port->ip_pset != IPS_NULL)
goto abort_request_send_receive;
rcv_object = (ipc_object_t) reply_port;
io_reference(rcv_object);
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
io_unlock(rcv_object);
goto fast_send_receive;
abort_request_copyin:
is_read_unlock(space);
goto slow_copyin;
abort_request_send_receive:
ip_unlock(dest_port);
ip_unlock(reply_port);
goto slow_send;
}
case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
ipc_entry_num_t size;
ipc_entry_t table;
/* sending a reply message */
{
mach_port_t reply_name =
kmsg->ikm_header.msgh_local_port;
if (reply_name != MACH_PORT_NULL)
goto slow_copyin;
}
is_write_lock(space);
assert(space->is_active);
/* optimized ipc_entry_lookup */
size = space->is_table_size;
table = space->is_table;
{
ipc_entry_t entry;
mach_port_gen_t gen;
mach_port_index_t index;
{
mach_port_t dest_name =
kmsg->ikm_header.msgh_remote_port;
index = MACH_PORT_INDEX(dest_name);
gen = MACH_PORT_GEN(dest_name);
}
if (index >= size)
goto abort_reply_dest_copyin;
entry = &table[index];
/* check generation, collision bit, and type bit */
if ((entry->ie_bits & (IE_BITS_GEN_MASK|
IE_BITS_COLLISION|
MACH_PORT_TYPE_SEND_ONCE)) !=
(gen | MACH_PORT_TYPE_SEND_ONCE))
goto abort_reply_dest_copyin;
/* optimized ipc_right_copyin */
assert(IE_BITS_TYPE(entry->ie_bits) ==
MACH_PORT_TYPE_SEND_ONCE);
assert(IE_BITS_UREFS(entry->ie_bits) == 1);
assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0);
if (entry->ie_request != 0)
goto abort_reply_dest_copyin;
dest_port = (ipc_port_t) entry->ie_object;
assert(dest_port != IP_NULL);
ip_lock(dest_port);
if (!ip_active(dest_port)) {
ip_unlock(dest_port);
goto abort_reply_dest_copyin;
}
assert(dest_port->ip_sorights > 0);
/* optimized ipc_entry_dealloc */
entry->ie_next = table->ie_next;
table->ie_next = index;
entry->ie_bits = gen;
entry->ie_object = IO_NULL;
}
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
0);
kmsg->ikm_header.msgh_remote_port =
(mach_port_t) dest_port;
/* make sure we can queue to the destination */
assert(dest_port->ip_receiver != ipc_space_kernel);
/* optimized ipc_entry_lookup/ipc_mqueue_copyin */
{
ipc_entry_t entry;
ipc_entry_bits_t bits;
{
mach_port_index_t index;
mach_port_gen_t gen;
index = MACH_PORT_INDEX(rcv_name);
gen = MACH_PORT_GEN(rcv_name);
if (index >= size)
goto abort_reply_rcv_copyin;
entry = &table[index];
bits = entry->ie_bits;
/* check generation number */
if ((bits & IE_BITS_GEN_MASK) != gen)
goto abort_reply_rcv_copyin;
}
/* check type bits; looking for receive or set */
if (bits & MACH_PORT_TYPE_PORT_SET) {
ipc_pset_t rcv_pset;
rcv_pset = (ipc_pset_t) entry->ie_object;
assert(rcv_pset != IPS_NULL);
ips_lock(rcv_pset);
assert(ips_active(rcv_pset));
rcv_object = (ipc_object_t) rcv_pset;
rcv_mqueue = &rcv_pset->ips_messages;
} else if (bits & MACH_PORT_TYPE_RECEIVE) {
ipc_port_t rcv_port;
rcv_port = (ipc_port_t) entry->ie_object;
assert(rcv_port != IP_NULL);
if (!ip_lock_try(rcv_port))
goto abort_reply_rcv_copyin;
assert(ip_active(rcv_port));
if (rcv_port->ip_pset != IPS_NULL) {
ip_unlock(rcv_port);
goto abort_reply_rcv_copyin;
}
rcv_object = (ipc_object_t) rcv_port;
rcv_mqueue = &rcv_port->ip_messages;
} else
goto abort_reply_rcv_copyin;
}
is_write_unlock(space);
io_reference(rcv_object);
imq_lock(rcv_mqueue);
io_unlock(rcv_object);
goto fast_send_receive;
abort_reply_dest_copyin:
is_write_unlock(space);
goto slow_copyin;
abort_reply_rcv_copyin:
ip_unlock(dest_port);
is_write_unlock(space);
goto slow_send;
}
default:
goto slow_copyin;
}
/*NOTREACHED*/
fast_send_receive:
/*
* optimized ipc_mqueue_send/ipc_mqueue_receive
*
* Finished get/copyin of kmsg and copyin of rcv_name.
* space is unlocked, dest_port is locked,
* we can queue kmsg to dest_port,
* rcv_mqueue is locked, rcv_object holds a ref,
* if rcv_object is a port it isn't in a port set
*
* Note that if simple locking is turned off,
* then we could have dest_mqueue == rcv_mqueue
* and not abort when we try to lock dest_mqueue.
*/
assert(ip_active(dest_port));
assert(dest_port->ip_receiver != ipc_space_kernel);
assert((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
(MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
MACH_MSG_TYPE_PORT_SEND_ONCE));
assert((kmsg->ikm_header.msgh_bits &
MACH_MSGH_BITS_CIRCULAR) == 0);
{
ipc_mqueue_t dest_mqueue;
ipc_thread_t receiver;
{
ipc_pset_t dest_pset;
dest_pset = dest_port->ip_pset;
if (dest_pset == IPS_NULL)
dest_mqueue = &dest_port->ip_messages;
else
dest_mqueue = &dest_pset->ips_messages;
}
if (!imq_lock_try(dest_mqueue)) {
abort_send_receive:
ip_unlock(dest_port);
imq_unlock(rcv_mqueue);
ipc_object_release(rcv_object);
goto slow_send;
}
receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
if ((receiver == ITH_NULL) ||
(ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
!= IKM_NULL)) {
imq_unlock(dest_mqueue);
goto abort_send_receive;
}
/*
* There is a receiver thread waiting, and
* there is no reply message for us to pick up.
* We have hope of hand-off, so save state.
*/
self->ith_msg = msg;
self->ith_rcv_size = rcv_size;
self->ith_object = rcv_object;
self->ith_mqueue = rcv_mqueue;
if ((receiver->swap_func == (void (*)()) mach_msg_continue) &&
thread_handoff(self, mach_msg_continue, receiver)) {
assert(current_thread() == receiver);
/*
* We can use the optimized receive code,
* because the receiver is using no options.
*/
} else if ((receiver->swap_func ==
(void (*)()) exception_raise_continue) &&
thread_handoff(self, mach_msg_continue, receiver)) {
counter(c_mach_msg_trap_block_exc++);
assert(current_thread() == receiver);
/*
* We are a reply message coming back through
* the optimized exception-handling path.
* Finish with rcv_mqueue and dest_mqueue,
* and then jump to exception code with
* dest_port still locked. We don't bother
* with a sequence number in this case.
*/
ipc_thread_enqueue_macro(
&rcv_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(rcv_mqueue);
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
imq_unlock(dest_mqueue);
exception_raise_continue_fast(dest_port, kmsg);
/*NOTREACHED*/
return MACH_MSG_SUCCESS;
} else if ((send_size <= receiver->ith_msize) &&
thread_handoff(self, mach_msg_continue, receiver)) {
assert(current_thread() == receiver);
if ((receiver->swap_func ==
(void (*)()) mach_msg_receive_continue) &&
((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) {
/*
* We can still use the optimized code.
*/
} else {
counter(c_mach_msg_trap_block_slow++);
/*
* We are running as the receiver,
* but we can't use the optimized code.
* Finish send/receive processing.
*/
dest_port->ip_msgcount++;
ip_unlock(dest_port);
ipc_thread_enqueue_macro(
&rcv_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(rcv_mqueue);
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
receiver->ith_state = MACH_MSG_SUCCESS;
receiver->ith_kmsg = kmsg;
receiver->ith_seqno = dest_port->ip_seqno++;
imq_unlock(dest_mqueue);
/*
* Call the receiver's continuation.
*/
receiver->wait_result = THREAD_AWAKENED;
(*receiver->swap_func)();
/*NOTREACHED*/
return MACH_MSG_SUCCESS;
}
} else {
/*
* The receiver can't accept the message,
* or we can't switch to the receiver.
*/
imq_unlock(dest_mqueue);
goto abort_send_receive;
}
counter(c_mach_msg_trap_block_fast++);
/*
* Safe to unlock dest_port now that we are
* committed to this path, because we hold
* dest_mqueue locked. We never bother changing
* dest_port->ip_msgcount.
*/
ip_unlock(dest_port);
/*
* We need to finish preparing self for its
* time asleep in rcv_mqueue.
*/
ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(rcv_mqueue);
/*
* Finish extracting receiver from dest_mqueue.
*/
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
imq_unlock(dest_mqueue);
/*
* We don't have to do any post-dequeue processing of
* the message. We never incremented ip_msgcount, we
* know it has no msg-accepted request, and blocked
* senders aren't a worry because we found the port
* with a receiver waiting.
*/
self = receiver;
space = self->task->itk_space;
msg = self->ith_msg;
rcv_size = self->ith_rcv_size;
rcv_object = self->ith_object;
/* inline ipc_object_release */
io_lock(rcv_object);
io_release(rcv_object);
io_check_unlock(rcv_object);
}
fast_copyout:
/*
* Nothing locked and no references held, except
* we have kmsg with msgh_seqno filled in. Must
* still check against rcv_size and do
* ipc_kmsg_copyout/ipc_kmsg_put.
*/
assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port
== dest_port);
reply_size = kmsg->ikm_header.msgh_size;
if (rcv_size < reply_size)
goto slow_copyout;
/* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
switch (kmsg->ikm_header.msgh_bits) {
case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE): {
ipc_port_t reply_port =
(ipc_port_t) kmsg->ikm_header.msgh_local_port;
mach_port_t dest_name, reply_name;
/* receiving a request message */
if (!IP_VALID(reply_port))
goto slow_copyout;
is_write_lock(space);
assert(space->is_active);
/*
* To do an atomic copyout, need simultaneous
* locks on both ports and the space. If
* dest_port == reply_port, and simple locking is
* enabled, then we will abort. Otherwise it's
* OK to unlock twice.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port))
goto abort_request_copyout;
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
goto abort_request_copyout;
}
assert(reply_port->ip_sorights > 0);
ip_unlock(reply_port);
{
ipc_entry_t table;
ipc_entry_t entry;
mach_port_index_t index;
/* optimized ipc_entry_get */
table = space->is_table;
index = table->ie_next;
if (index == 0)
goto abort_request_copyout;
entry = &table[index];
table->ie_next = entry->ie_next;
entry->ie_request = 0;
{
mach_port_gen_t gen;
assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
gen = entry->ie_bits + IE_BITS_GEN_ONE;
reply_name = MACH_PORT_MAKE(index, gen);
/* optimized ipc_right_copyout */
entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
}
assert(MACH_PORT_VALID(reply_name));
entry->ie_object = (ipc_object_t) reply_port;
is_write_unlock(space);
}
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_srights > 0);
ip_release(dest_port);
if (dest_port->ip_receiver == space)
dest_name = dest_port->ip_receiver_name;
else
dest_name = MACH_PORT_NULL;
if ((--dest_port->ip_srights == 0) &&
(dest_port->ip_nsrequest != IP_NULL)) {
ipc_port_t nsrequest;
mach_port_mscount_t mscount;
/* a rather rare case */
nsrequest = dest_port->ip_nsrequest;
mscount = dest_port->ip_mscount;
dest_port->ip_nsrequest = IP_NULL;
ip_unlock(dest_port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(dest_port);
if (! ipc_port_flag_protected_payload(dest_port)) {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PORT_SEND);
kmsg->ikm_header.msgh_local_port = dest_name;
} else {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PROTECTED_PAYLOAD);
kmsg->ikm_header.msgh_protected_payload =
dest_port->ip_protected_payload;
}
kmsg->ikm_header.msgh_remote_port = reply_name;
goto fast_put;
abort_request_copyout:
ip_unlock(dest_port);
is_write_unlock(space);
goto slow_copyout;
}
case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
mach_port_t dest_name;
/* receiving a reply message */
ip_lock(dest_port);
if (!ip_active(dest_port))
goto slow_copyout;
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_sorights > 0);
if (dest_port->ip_receiver == space) {
ip_release(dest_port);
dest_port->ip_sorights--;
dest_name = dest_port->ip_receiver_name;
ip_unlock(dest_port);
} else {
ip_unlock(dest_port);
ipc_notify_send_once(dest_port);
dest_name = MACH_PORT_NULL;
}
if (! ipc_port_flag_protected_payload(dest_port)) {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PORT_SEND_ONCE);
kmsg->ikm_header.msgh_local_port = dest_name;
} else {
kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PROTECTED_PAYLOAD);
kmsg->ikm_header.msgh_protected_payload =
dest_port->ip_protected_payload;
}
kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
goto fast_put;
}
case MACH_MSGH_BITS_COMPLEX|
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
mach_port_t dest_name;
/* receiving a complex reply message */
ip_lock(dest_port);
if (!ip_active(dest_port))
goto slow_copyout;
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_sorights > 0);
if (dest_port->ip_receiver == space) {
ip_release(dest_port);
dest_port->ip_sorights--;
dest_name = dest_port->ip_receiver_name;
ip_unlock(dest_port);
} else {
ip_unlock(dest_port);
ipc_notify_send_once(dest_port);
dest_name = MACH_PORT_NULL;
}
if (! ipc_port_flag_protected_payload(dest_port)) {
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS_COMPLEX
| MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PORT_SEND_ONCE);
kmsg->ikm_header.msgh_local_port = dest_name;
} else {
kmsg->ikm_header.msgh_bits =
MACH_MSGH_BITS_COMPLEX
| MACH_MSGH_BITS(
0,
MACH_MSG_TYPE_PROTECTED_PAYLOAD);
kmsg->ikm_header.msgh_protected_payload =
dest_port->ip_protected_payload;
}
kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
mr = ipc_kmsg_copyout_body(
(vm_offset_t) (&kmsg->ikm_header + 1),
(vm_offset_t) &kmsg->ikm_header
+ kmsg->ikm_header.msgh_size,
space,
current_map());
if (mr != MACH_MSG_SUCCESS) {
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
return mr | MACH_RCV_BODY_ERROR;
}
goto fast_put;
}
default:
goto slow_copyout;
}
/*NOTREACHED*/
fast_put:
/*
* We have the reply message data in kmsg,
* and the reply message size in reply_size.
* Just need to copy it out to the user and free kmsg.
* We must check ikm_cache after copyoutmsg.
*/
ikm_check_initialized(kmsg, kmsg->ikm_size);
if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
copyoutmsg(&kmsg->ikm_header, msg,
reply_size) ||
(ikm_cache() != IKM_NULL))
goto slow_put;
ikm_cache() = kmsg;
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
return MACH_MSG_SUCCESS; /* help for the compiler */
/*
* The slow path has a few non-register temporary
* variables used only for call-by-reference.
*/
{
ipc_kmsg_t temp_kmsg;
mach_port_seqno_t temp_seqno;
ipc_object_t temp_rcv_object;
ipc_mqueue_t temp_rcv_mqueue;
slow_get:
/*
* No locks, references, or messages held.
* Still have to get the request, send it,
* receive reply, etc.
*/
mr = ipc_kmsg_get(msg, send_size, &temp_kmsg);
if (mr != MACH_MSG_SUCCESS) {
thread_syscall_return(mr);
/*NOTREACHED*/
}
kmsg = temp_kmsg;
/* try to get back on optimized path */
goto fast_copyin;
slow_copyin:
/*
* We have the message data in kmsg, but
* we still need to copyin, send it,
* receive a reply, and do copyout.
*/
mr = ipc_kmsg_copyin(kmsg, space, current_map(),
MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ikm_free(kmsg);
thread_syscall_return(mr);
/*NOTREACHED*/
}
/* try to get back on optimized path */
if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR)
goto slow_send;
dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
assert(IP_VALID(dest_port));
ip_lock(dest_port);
if (dest_port->ip_receiver == ipc_space_kernel) {
assert(ip_active(dest_port));
ip_unlock(dest_port);
goto kernel_send;
}
if (ip_active(dest_port) &&
((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
(MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
MACH_MSG_TYPE_PORT_SEND_ONCE)))
{
/*
* Try an optimized ipc_mqueue_copyin.
* It will work if this is a request message.
*/
ipc_port_t reply_port;
reply_port = (ipc_port_t)
kmsg->ikm_header.msgh_local_port;
if (IP_VALID(reply_port)) {
if (ip_lock_try(reply_port)) {
if (ip_active(reply_port) &&
reply_port->ip_receiver == space &&
reply_port->ip_receiver_name == rcv_name &&
reply_port->ip_pset == IPS_NULL)
{
/* Grab a reference to the reply port. */
rcv_object = (ipc_object_t) reply_port;
io_reference(rcv_object);
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
io_unlock(rcv_object);
goto fast_send_receive;
}
ip_unlock(reply_port);
}
}
}
ip_unlock(dest_port);
goto slow_send;
kernel_send:
/*
* Special case: send message to kernel services.
* The request message has been copied into the
* kmsg. Nothing is locked.
*/
{
ipc_port_t reply_port;
/*
* Perform the kernel function.
*/
kmsg = ipc_kobject_server(kmsg);
if (kmsg == IKM_NULL) {
/*
* No reply. Take the
* slow receive path.
*/
goto slow_get_rcv_port;
}
/*
* Check that:
* the reply port is alive
* we hold the receive right
* the name has not changed.
* the port is not in a set
* If any of these are not true,
* we cannot directly receive the reply
* message.
*/
reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
ip_lock(reply_port);
if ((!ip_active(reply_port)) ||
(reply_port->ip_receiver != space) ||
(reply_port->ip_receiver_name != rcv_name) ||
(reply_port->ip_pset != IPS_NULL))
{
ip_unlock(reply_port);
ipc_mqueue_send_always(kmsg);
goto slow_get_rcv_port;
}
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
/* keep port locked, and don`t change ref count yet */
/*
* If there are messages on the port
* or other threads waiting for a message,
* we cannot directly receive the reply.
*/
if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads)
!= ITH_NULL) ||
(ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
!= IKM_NULL))
{
imq_unlock(rcv_mqueue);
ip_unlock(reply_port);
ipc_mqueue_send_always(kmsg);
goto slow_get_rcv_port;
}
/*
* We can directly receive this reply.
* Since the kernel reply never blocks,
* it holds no message_accepted request.
* Since there were no messages queued
* on the reply port, there should be
* no threads blocked waiting to send.
*/
assert(kmsg->ikm_marequest == IMAR_NULL);
assert(ipc_thread_queue_first(&reply_port->ip_blocked)
== ITH_NULL);
dest_port = reply_port;
kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
imq_unlock(rcv_mqueue);
/*
* inline ipc_object_release.
* Port is still locked.
* Reference count was not incremented.
*/
ip_check_unlock(reply_port);
/* copy out the kernel reply */
goto fast_copyout;
}
slow_send:
/*
* Nothing is locked. We have acquired kmsg, but
* we still need to send it and receive a reply.
*/
mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
current_map());
assert(kmsg->ikm_marequest == IMAR_NULL);
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
slow_get_rcv_port:
/*
* We have sent the message. Copy in the receive port.
*/
mr = ipc_mqueue_copyin(space, rcv_name,
&temp_rcv_mqueue, &temp_rcv_object);
if (mr != MACH_MSG_SUCCESS) {
thread_syscall_return(mr);
/*NOTREACHED*/
}
rcv_mqueue = temp_rcv_mqueue;
rcv_object = temp_rcv_object;
/* hold ref for rcv_object; rcv_mqueue is locked */
/*
slow_receive:
*/
/*
* Now we have sent the request and copied in rcv_name,
* so rcv_mqueue is locked and hold ref for rcv_object.
* Just receive a reply and try to get back to fast path.
*
* ipc_mqueue_receive may not return, because if we block
* then our kernel stack may be discarded. So we save
* state here for mach_msg_continue to pick up.
*/
self->ith_msg = msg;
self->ith_rcv_size = rcv_size;
self->ith_object = rcv_object;
self->ith_mqueue = rcv_mqueue;
mr = ipc_mqueue_receive(rcv_mqueue,
MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, mach_msg_continue,
&temp_kmsg, &temp_seqno);
/* rcv_mqueue is unlocked */
ipc_object_release(rcv_object);
if (mr != MACH_MSG_SUCCESS) {
thread_syscall_return(mr);
/*NOTREACHED*/
}
(kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno;
dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
goto fast_copyout;
slow_copyout:
/*
* Nothing locked and no references held, except
* we have kmsg with msgh_seqno filled in. Must
* still check against rcv_size and do
* ipc_kmsg_copyout/ipc_kmsg_put.
*/
reply_size = kmsg->ikm_header.msgh_size;
if (rcv_size < reply_size) {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
thread_syscall_return(MACH_RCV_TOO_LARGE);
/*NOTREACHED*/
}
mr = ipc_kmsg_copyout(kmsg, space, current_map(),
MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
} else {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
}
thread_syscall_return(mr);
/*NOTREACHED*/
}
/* try to get back on optimized path */
goto fast_put;
slow_put:
mr = ipc_kmsg_put(msg, kmsg, reply_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
} else if (option == MACH_SEND_MSG) {
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_kmsg_t kmsg;
mr = ipc_kmsg_get(msg, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ikm_free(kmsg);
return mr;
}
mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
assert(kmsg->ikm_marequest == IMAR_NULL);
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
}
return mr;
} else if (option == MACH_RCV_MSG) {
ipc_thread_t self = current_thread();
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_object_t object;
ipc_mqueue_t mqueue;
ipc_kmsg_t kmsg;
mach_port_seqno_t seqno;
mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
if (mr != MACH_MSG_SUCCESS)
return mr;
/* hold ref for object; mqueue is locked */
/*
* ipc_mqueue_receive may not return, because if we block
* then our kernel stack may be discarded. So we save
* state here for mach_msg_continue to pick up.
*/
self->ith_msg = msg;
self->ith_rcv_size = rcv_size;
self->ith_object = object;
self->ith_mqueue = mqueue;
mr = ipc_mqueue_receive(mqueue,
MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, mach_msg_continue,
&kmsg, &seqno);
/* mqueue is unlocked */
ipc_object_release(object);
if (mr != MACH_MSG_SUCCESS)
return mr;
kmsg->ikm_header.msgh_seqno = seqno;
if (rcv_size < kmsg->ikm_header.msgh_size) {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
return MACH_RCV_TOO_LARGE;
}
mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
(void) ipc_kmsg_put(msg, kmsg,
kmsg->ikm_header.msgh_size);
} else {
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
}
return mr;
}
return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
} else if (option == MACH_MSG_OPTION_NONE) {
/*
* We can measure the "null mach_msg_trap"
* (syscall entry and thread_syscall_return exit)
* with this path.
*/
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
}
if (option & MACH_SEND_MSG) {
mr = mach_msg_send(msg, option, send_size,
time_out, notify);
if (mr != MACH_MSG_SUCCESS)
return mr;
}
if (option & MACH_RCV_MSG) {
mr = mach_msg_receive(msg, option, rcv_size, rcv_name,
time_out, notify);
if (mr != MACH_MSG_SUCCESS)
return mr;
}
return MACH_MSG_SUCCESS;
}
kern_return_t
ipc_object_copyout_name(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
boolean_t overflow,
mach_port_name_t name)
{
mach_port_name_t oname;
ipc_entry_t oentry;
ipc_entry_t entry;
kern_return_t kr;
#if IMPORTANCE_INHERITANCE
int assertcnt = 0;
ipc_importance_task_t task_imp = IIT_NULL;
#endif /* IMPORTANCE_INHERITANCE */
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
kr = ipc_entry_alloc_name(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
if ((msgt_name != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
ipc_right_reverse(space, object, &oname, &oentry)) {
/* object is locked and active */
if (name != oname) {
io_unlock(object);
if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
return KERN_RIGHT_EXISTS;
}
assert(entry == oentry);
assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
} else {
if (ipc_right_inuse(space, name, entry))
return KERN_NAME_EXISTS;
assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
assert(entry->ie_object == IO_NULL);
io_lock(object);
if (!io_active(object)) {
io_unlock(object);
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
return KERN_INVALID_CAPABILITY;
}
entry->ie_object = object;
}
/* space is write-locked and active, object is locked and active */
#if IMPORTANCE_INHERITANCE
/*
* We are slamming a receive right into the space, without
* first having been enqueued on a port destined there. So,
* we have to arrange to boost the task appropriately if this
* port has assertions (and the task wants them).
*/
if (msgt_name == MACH_MSG_TYPE_PORT_RECEIVE) {
ipc_port_t port = (ipc_port_t)object;
if (space->is_task != TASK_NULL) {
task_imp = space->is_task->task_imp_base;
if (ipc_importance_task_is_any_receiver_type(task_imp)) {
assertcnt = port->ip_impcount;
ipc_importance_task_reference(task_imp);
}
}
/* take port out of limbo */
assert(port->ip_tempowner != 0);
port->ip_tempowner = 0;
}
#endif /* IMPORTANCE_INHERITANCE */
kr = ipc_right_copyout(space, name, entry,
msgt_name, overflow, object);
/* object is unlocked */
is_write_unlock(space);
#if IMPORTANCE_INHERITANCE
/*
* Add the assertions to the task that we captured before
*/
if (task_imp != IIT_NULL) {
ipc_importance_task_hold_internal_assertion(task_imp, assertcnt);
ipc_importance_task_release(task_imp);
}
#endif /* IMPORTANCE_INHERITANCE */
return kr;
}
kern_return_t
ipc_object_copyout(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
boolean_t overflow,
mach_port_name_t *namep)
{
mach_port_name_t name;
ipc_entry_t entry;
kern_return_t kr;
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
is_write_lock(space);
for (;;) {
if (!is_active(space)) {
is_write_unlock(space);
return KERN_INVALID_TASK;
}
if ((msgt_name != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
ipc_right_reverse(space, object, &name, &entry)) {
/* object is locked and active */
assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
break;
}
name = CAST_MACH_PORT_TO_NAME(object);
kr = ipc_entry_get(space, &name, &entry);
if (kr != KERN_SUCCESS) {
/* unlocks/locks space, so must start again */
kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
if (kr != KERN_SUCCESS)
return kr; /* space is unlocked */
continue;
}
assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
assert(entry->ie_object == IO_NULL);
io_lock(object);
if (!io_active(object)) {
io_unlock(object);
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
return KERN_INVALID_CAPABILITY;
}
entry->ie_object = object;
break;
}
/* space is write-locked and active, object is locked and active */
kr = ipc_right_copyout(space, name, entry,
msgt_name, overflow, object);
/* object is unlocked */
is_write_unlock(space);
if (kr == KERN_SUCCESS)
*namep = name;
return kr;
}
kern_return_t
ipc_right_dnrequest(
ipc_space_t space,
mach_port_t name,
boolean_t immediate,
ipc_port_t notify,
ipc_port_t *previousp)
{
ipc_port_t previous;
for (;;) {
ipc_entry_t entry;
ipc_entry_bits_t bits;
kern_return_t kr;
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
bits = entry->ie_bits;
if (bits & MACH_PORT_TYPE_PORT_RIGHTS) {
ipc_port_t port;
ipc_port_request_index_t request;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (!ipc_right_check(space, port, name, entry)) {
/* port is locked and active */
if (notify == IP_NULL) {
previous = ipc_right_dncancel_macro(
space, port, name, entry);
ip_unlock(port);
is_write_unlock(space);
break;
}
/*
* If a registered soright exists,
* want to atomically switch with it.
* If ipc_port_dncancel finds us a
* soright, then the following
* ipc_port_dnrequest will reuse
* that slot, so we are guaranteed
* not to unlock and retry.
*/
previous = ipc_right_dncancel_macro(space,
port, name, entry);
kr = ipc_port_dnrequest(port, name, notify,
&request);
if (kr != KERN_SUCCESS) {
assert(previous == IP_NULL);
is_write_unlock(space);
kr = ipc_port_dngrow(port);
/* port is unlocked */
if (kr != KERN_SUCCESS)
return kr;
continue;
}
assert(request != 0);
ip_unlock(port);
entry->ie_request = request;
is_write_unlock(space);
break;
}
bits = entry->ie_bits;
assert(bits & MACH_PORT_TYPE_DEAD_NAME);
}
if ((bits & MACH_PORT_TYPE_DEAD_NAME) &&
immediate && (notify != IP_NULL)) {
mach_port_urefs_t urefs = IE_BITS_UREFS(bits);
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
assert(urefs > 0);
if (MACH_PORT_UREFS_OVERFLOW(urefs, 1)) {
is_write_unlock(space);
return KERN_UREFS_OVERFLOW;
}
entry->ie_bits = bits + 1; /* increment urefs */
is_write_unlock(space);
ipc_notify_dead_name(notify, name);
previous = IP_NULL;
break;
}
is_write_unlock(space);
if (bits & MACH_PORT_TYPE_PORT_OR_DEAD)
return KERN_INVALID_ARGUMENT;
else
return KERN_INVALID_RIGHT;
}
*previousp = previous;
return KERN_SUCCESS;
}
kern_return_t
ipc_object_copyout_name(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
boolean_t overflow,
mach_port_name_t name)
{
mach_port_name_t oname;
ipc_entry_t oentry;
ipc_entry_t entry;
kern_return_t kr;
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
kr = ipc_entry_alloc_name(space, name, &entry);
if (kr != KERN_SUCCESS)
return kr;
/* space is write-locked and active */
if ((msgt_name != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
ipc_right_reverse(space, object, &oname, &oentry)) {
/* object is locked and active */
if (name != oname) {
io_unlock(object);
if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
return KERN_RIGHT_EXISTS;
}
assert(entry == oentry);
assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
} else {
if (ipc_right_inuse(space, name, entry))
return KERN_NAME_EXISTS;
assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
assert(entry->ie_object == IO_NULL);
io_lock(object);
if (!io_active(object)) {
io_unlock(object);
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
return KERN_INVALID_CAPABILITY;
}
entry->ie_object = object;
}
/* space is write-locked and active, object is locked and active */
kr = ipc_right_copyout(space, name, entry,
msgt_name, overflow, object);
/* object is unlocked */
is_write_unlock(space);
return kr;
}
kern_return_t
ipc_right_dealloc(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry)
{
ipc_entry_bits_t bits = entry->ie_bits;
mach_port_type_t type = IE_BITS_TYPE(bits);
assert(space->is_active);
switch (type) {
case MACH_PORT_TYPE_DEAD_NAME: {
dead_name:
assert(IE_BITS_UREFS(bits) > 0);
assert(entry->ie_request == 0);
assert(entry->ie_object == IO_NULL);
assert((bits & IE_BITS_MAREQUEST) == 0);
if (IE_BITS_UREFS(bits) == 1)
ipc_entry_dealloc(space, name, entry);
else
entry->ie_bits = bits-1; /* decrement urefs */
is_write_unlock(space);
break;
}
case MACH_PORT_TYPE_SEND_ONCE: {
ipc_port_t port, dnrequest;
assert(IE_BITS_UREFS(bits) == 1);
assert((bits & IE_BITS_MAREQUEST) == 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
goto dead_name;
}
/* port is locked and active */
assert(port->ip_sorights > 0);
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
ip_unlock(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
ipc_notify_send_once(port);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_TYPE_SEND: {
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
bits = entry->ie_bits;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
goto dead_name;
}
/* port is locked and active */
assert(port->ip_srights > 0);
if (IE_BITS_UREFS(bits) == 1) {
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
dnrequest = ipc_right_dncancel_macro(space, port,
name, entry);
ipc_reverse_remove(space, (ipc_object_t) port);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
ip_release(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
} else
entry->ie_bits = bits-1; /* decrement urefs */
ip_unlock(port); /* even if dropped a ref, port is active */
is_write_unlock(space);
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_TYPE_SEND_RECEIVE: {
ipc_port_t port;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
assert(IE_BITS_UREFS(bits) > 0);
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
assert(port->ip_srights > 0);
if (IE_BITS_UREFS(bits) == 1) {
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK|
MACH_PORT_TYPE_SEND);
} else
entry->ie_bits = bits-1; /* decrement urefs */
ip_unlock(port);
is_write_unlock(space);
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
break;
}
default:
is_write_unlock(space);
return KERN_INVALID_RIGHT;
}
return KERN_SUCCESS;
}
kern_return_t
ipc_right_delta(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry,
mach_port_right_t right,
mach_port_delta_t delta)
{
ipc_entry_bits_t bits = entry->ie_bits;
assert(space->is_active);
assert(right < MACH_PORT_RIGHT_NUMBER);
/* Rights-specific restrictions and operations. */
switch (right) {
case MACH_PORT_RIGHT_PORT_SET: {
ipc_pset_t pset;
if ((bits & MACH_PORT_TYPE_PORT_SET) == 0)
goto invalid_right;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_PORT_SET);
assert(IE_BITS_UREFS(bits) == 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(entry->ie_request == 0);
if (delta == 0)
goto success;
if (delta != -1)
goto invalid_value;
pset = (ipc_pset_t) entry->ie_object;
assert(pset != IPS_NULL);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
ips_lock(pset);
assert(ips_active(pset));
is_write_unlock(space);
ipc_pset_destroy(pset); /* consumes ref, unlocks */
break;
}
case MACH_PORT_RIGHT_RECEIVE: {
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
goto invalid_right;
if (delta == 0)
goto success;
if (delta != -1)
goto invalid_value;
if (bits & IE_BITS_MAREQUEST) {
bits &= ~IE_BITS_MAREQUEST;
ipc_marequest_cancel(space, name);
}
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
/*
* The port lock is needed for ipc_right_dncancel;
* otherwise, we wouldn't have to take the lock
* until just before dropping the space lock.
*/
ip_lock(port);
assert(ip_active(port));
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
if (bits & MACH_PORT_TYPE_SEND) {
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
assert(IE_BITS_UREFS(bits) > 0);
assert(IE_BITS_UREFS(bits) < MACH_PORT_UREFS_MAX);
assert(port->ip_srights > 0);
/*
* The remaining send right turns into a
* dead name. Notice we don't decrement
* ip_srights, generate a no-senders notif,
* or use ipc_right_dncancel, because the
* port is destroyed "first".
*/
bits &= ~IE_BITS_TYPE_MASK;
bits |= MACH_PORT_TYPE_DEAD_NAME;
if (entry->ie_request != 0) {
entry->ie_request = 0;
bits++; /* increment urefs */
}
entry->ie_bits = bits;
entry->ie_object = IO_NULL;
} else {
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE);
assert(IE_BITS_UREFS(bits) == 0);
dnrequest = ipc_right_dncancel_macro(space, port,
name, entry);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
}
is_write_unlock(space);
ipc_port_clear_receiver(port);
ipc_port_destroy(port); /* consumes ref, unlocks */
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_RIGHT_SEND_ONCE: {
ipc_port_t port, dnrequest;
if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0)
goto invalid_right;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
assert(IE_BITS_UREFS(bits) == 1);
assert((bits & IE_BITS_MAREQUEST) == 0);
if ((delta > 0) || (delta < -1))
goto invalid_value;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
assert(!(entry->ie_bits & MACH_PORT_TYPE_SEND_ONCE));
goto invalid_right;
}
/* port is locked and active */
assert(port->ip_sorights > 0);
if (delta == 0) {
ip_unlock(port);
goto success;
}
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
ip_unlock(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
ipc_notify_send_once(port);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
case MACH_PORT_RIGHT_DEAD_NAME: {
mach_port_urefs_t urefs;
if (bits & MACH_PORT_TYPE_SEND_RIGHTS) {
ipc_port_t port;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (!ipc_right_check(space, port, name, entry)) {
/* port is locked and active */
ip_unlock(port);
goto invalid_right;
}
bits = entry->ie_bits;
} else if ((bits & MACH_PORT_TYPE_DEAD_NAME) == 0)
goto invalid_right;
assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME);
assert(IE_BITS_UREFS(bits) > 0);
assert((bits & IE_BITS_MAREQUEST) == 0);
assert(entry->ie_object == IO_NULL);
assert(entry->ie_request == 0);
urefs = IE_BITS_UREFS(bits);
if (MACH_PORT_UREFS_UNDERFLOW(urefs, delta))
goto invalid_value;
if (MACH_PORT_UREFS_OVERFLOW(urefs, delta))
goto urefs_overflow;
if ((urefs + delta) == 0)
ipc_entry_dealloc(space, name, entry);
else
entry->ie_bits = bits + delta;
is_write_unlock(space);
break;
}
case MACH_PORT_RIGHT_SEND: {
mach_port_urefs_t urefs;
ipc_port_t port;
ipc_port_t dnrequest = IP_NULL;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
if ((bits & MACH_PORT_TYPE_SEND) == 0)
goto invalid_right;
/* maximum urefs for send is MACH_PORT_UREFS_MAX-1 */
urefs = IE_BITS_UREFS(bits);
if (MACH_PORT_UREFS_UNDERFLOW(urefs, delta))
goto invalid_value;
if (MACH_PORT_UREFS_OVERFLOW(urefs+1, delta))
goto urefs_overflow;
port = (ipc_port_t) entry->ie_object;
assert(port != IP_NULL);
if (ipc_right_check(space, port, name, entry)) {
assert((entry->ie_bits & MACH_PORT_TYPE_SEND) == 0);
goto invalid_right;
}
/* port is locked and active */
assert(port->ip_srights > 0);
if ((urefs + delta) == 0) {
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
if (bits & MACH_PORT_TYPE_RECEIVE) {
assert(port->ip_receiver_name == name);
assert(port->ip_receiver == space);
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND_RECEIVE);
entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK|
MACH_PORT_TYPE_SEND);
} else {
assert(IE_BITS_TYPE(bits) ==
MACH_PORT_TYPE_SEND);
dnrequest = ipc_right_dncancel_macro(
space, port, name, entry);
ipc_reverse_remove(space, (ipc_object_t) port);
if (bits & IE_BITS_MAREQUEST)
ipc_marequest_cancel(space, name);
ip_release(port);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
}
} else
entry->ie_bits = bits + delta;
ip_unlock(port); /* even if dropped a ref, port is active */
is_write_unlock(space);
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_delta: strange right");
#else
panic("ipc_right_delta: strange right");
#endif
}
return KERN_SUCCESS;
success:
is_write_unlock(space);
return KERN_SUCCESS;
invalid_right:
is_write_unlock(space);
return KERN_INVALID_RIGHT;
invalid_value:
is_write_unlock(space);
return KERN_INVALID_VALUE;
urefs_overflow:
is_write_unlock(space);
return KERN_UREFS_OVERFLOW;
}
void
ipc_space_destroy(
ipc_space_t space)
{
ipc_tree_entry_t tentry;
ipc_entry_t table;
ipc_entry_num_t size;
mach_port_index_t index;
boolean_t active;
assert(space != IS_NULL);
is_write_lock(space);
active = space->is_active;
space->is_active = FALSE;
is_write_unlock(space);
if (!active)
return;
/*
* If somebody is trying to grow the table,
* we must wait until they finish and figure
* out the space died.
*/
is_read_lock(space);
while (space->is_growing) {
assert_wait((event_t) space, FALSE);
is_read_unlock(space);
thread_block((void (*)(void)) 0);
is_read_lock(space);
}
is_read_unlock(space);
/*
* Now we can futz with it without having it locked.
*/
table = space->is_table;
size = space->is_table_size;
for (index = 0; index < size; index++) {
ipc_entry_t entry = &table[index];
mach_port_type_t type = IE_BITS_TYPE(entry->ie_bits);
if (type != MACH_PORT_TYPE_NONE) {
mach_port_t name =
MACH_PORT_MAKEB(index, entry->ie_bits);
ipc_right_clean(space, name, entry);
}
}
it_entries_free(space->is_table_next-1, table);
for (tentry = ipc_splay_traverse_start(&space->is_tree);
tentry != ITE_NULL;
tentry = ipc_splay_traverse_next(&space->is_tree, TRUE)) {
mach_port_type_t type = IE_BITS_TYPE(tentry->ite_bits);
mach_port_t name = tentry->ite_name;
assert(type != MACH_PORT_TYPE_NONE);
/* use object before ipc_right_clean releases ref */
if (type == MACH_PORT_TYPE_SEND)
ipc_hash_global_delete(space, tentry->ite_object,
name, tentry);
ipc_right_clean(space, name, &tentry->ite_entry);
}
ipc_splay_traverse_finish(&space->is_tree);
/*
* Because the space is now dead,
* we must release the "active" reference for it.
* Our caller still has his reference.
*/
is_release(space);
}
kern_return_t
ipc_right_destroy(
ipc_space_t space,
mach_port_t name,
ipc_entry_t entry)
{
ipc_entry_bits_t bits = entry->ie_bits;
mach_port_type_t type = IE_BITS_TYPE(bits);
assert(space->is_active);
switch (type) {
case MACH_PORT_TYPE_DEAD_NAME:
assert(entry->ie_request == 0);
assert(entry->ie_object == IO_NULL);
assert((bits & IE_BITS_MAREQUEST) == 0);
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
break;
case MACH_PORT_TYPE_PORT_SET: {
ipc_pset_t pset = (ipc_pset_t) entry->ie_object;
assert(entry->ie_request == 0);
assert(pset != IPS_NULL);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
ips_lock(pset);
assert(ips_active(pset));
is_write_unlock(space);
ipc_pset_destroy(pset); /* consumes ref, unlocks */
break;
}
case MACH_PORT_TYPE_SEND:
case MACH_PORT_TYPE_RECEIVE:
case MACH_PORT_TYPE_SEND_RECEIVE:
case MACH_PORT_TYPE_SEND_ONCE: {
ipc_port_t port = (ipc_port_t) entry->ie_object;
ipc_port_t nsrequest = IP_NULL;
mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */
ipc_port_t dnrequest;
assert(port != IP_NULL);
if (bits & IE_BITS_MAREQUEST) {
assert(type & MACH_PORT_TYPE_SEND_RECEIVE);
ipc_marequest_cancel(space, name);
}
if (type == MACH_PORT_TYPE_SEND)
ipc_reverse_remove(space, (ipc_object_t) port);
ip_lock(port);
if (!ip_active(port)) {
assert((type & MACH_PORT_TYPE_RECEIVE) == 0);
ip_release(port);
ip_check_unlock(port);
entry->ie_request = 0;
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
break;
}
dnrequest = ipc_right_dncancel_macro(space, port, name, entry);
entry->ie_object = IO_NULL;
ipc_entry_dealloc(space, name, entry);
is_write_unlock(space);
if (type & MACH_PORT_TYPE_SEND) {
assert(port->ip_srights > 0);
if (--port->ip_srights == 0) {
nsrequest = port->ip_nsrequest;
if (nsrequest != IP_NULL) {
port->ip_nsrequest = IP_NULL;
mscount = port->ip_mscount;
}
}
}
if (type & MACH_PORT_TYPE_RECEIVE) {
assert(ip_active(port));
assert(port->ip_receiver == space);
ipc_port_clear_receiver(port);
ipc_port_destroy(port); /* consumes our ref, unlocks */
} else if (type & MACH_PORT_TYPE_SEND_ONCE) {
assert(port->ip_sorights > 0);
ip_unlock(port);
ipc_notify_send_once(port); /* consumes our ref */
} else {
assert(port->ip_receiver != space);
ip_release(port);
ip_unlock(port);
}
if (nsrequest != IP_NULL)
ipc_notify_no_senders(nsrequest, mscount);
if (dnrequest != IP_NULL)
ipc_notify_port_deleted(dnrequest, name);
break;
}
default:
#if MACH_ASSERT
assert(!"ipc_right_destroy: strange type");
#else
panic("ipc_right_destroy: strange type");
#endif
}
return KERN_SUCCESS;
}
void
exception_raise(
ipc_port_t dest_port,
ipc_port_t thread_port,
ipc_port_t task_port,
integer_t _exception,
integer_t code,
integer_t subcode)
{
ipc_thread_t self = current_thread();
ipc_thread_t receiver;
ipc_port_t reply_port;
ipc_mqueue_t dest_mqueue;
ipc_mqueue_t reply_mqueue;
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
assert(IP_VALID(dest_port));
/*
* We will eventually need a message buffer.
* Grab the buffer now, while nothing is locked.
* This buffer will get handed to the exception server,
* and it will give the buffer back with its reply.
*/
kmsg = ikm_cache();
if (kmsg != IKM_NULL) {
ikm_cache() = IKM_NULL;
ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
} else {
kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
if (kmsg == IKM_NULL)
panic("exception_raise");
ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
}
/*
* We need a reply port for the RPC.
* Check first for a cached port.
*/
ith_lock(self);
assert(self->ith_self != IP_NULL);
reply_port = self->ith_rpc_reply;
if (reply_port == IP_NULL) {
ith_unlock(self);
reply_port = ipc_port_alloc_reply();
ith_lock(self);
if ((reply_port == IP_NULL) ||
(self->ith_rpc_reply != IP_NULL))
panic("exception_raise");
self->ith_rpc_reply = reply_port;
}
ip_lock(reply_port);
assert(ip_active(reply_port));
ith_unlock(self);
/*
* Make a naked send-once right for the reply port,
* to hand to the exception server.
* Make an extra reference for the reply port,
* to receive on. This protects us against
* mach_msg_abort_rpc.
*/
reply_port->ip_sorights++;
ip_reference(reply_port);
ip_reference(reply_port);
self->ith_port = reply_port;
reply_mqueue = &reply_port->ip_messages;
imq_lock(reply_mqueue);
assert(ipc_kmsg_queue_empty(&reply_mqueue->imq_messages));
ip_unlock(reply_port);
/*
* Make sure we can queue to the destination port.
*/
if (!ip_lock_try(dest_port)) {
imq_unlock(reply_mqueue);
goto slow_exception_raise;
}
if (!ip_active(dest_port) ||
(dest_port->ip_receiver == ipc_space_kernel)) {
imq_unlock(reply_mqueue);
ip_unlock(dest_port);
goto slow_exception_raise;
}
/*
* Find the destination message queue.
*/
{
ipc_pset_t dest_pset;
dest_pset = dest_port->ip_pset;
if (dest_pset == IPS_NULL)
dest_mqueue = &dest_port->ip_messages;
else
dest_mqueue = &dest_pset->ips_messages;
}
if (!imq_lock_try(dest_mqueue)) {
imq_unlock(reply_mqueue);
ip_unlock(dest_port);
goto slow_exception_raise;
}
/*
* Safe to unlock dest_port, because we hold
* dest_mqueue locked. We never bother changing
* dest_port->ip_msgcount.
*/
ip_unlock(dest_port);
receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
if ((receiver == ITH_NULL) ||
!((receiver->swap_func == (void (*)()) mach_msg_continue) ||
((receiver->swap_func ==
(void (*)()) mach_msg_receive_continue) &&
(sizeof(struct mach_exception) <= receiver->ith_msize) &&
((receiver->ith_option & MACH_RCV_NOTIFY) == 0))) ||
!thread_handoff(self, exception_raise_continue, receiver)) {
imq_unlock(reply_mqueue);
imq_unlock(dest_mqueue);
goto slow_exception_raise;
}
counter(c_exception_raise_block++);
assert(current_thread() == receiver);
/*
* We need to finish preparing self for its
* time asleep in reply_mqueue. self is left
* holding the extra ref for reply_port.
*/
ipc_thread_enqueue_macro(&reply_mqueue->imq_threads, self);
self->ith_state = MACH_RCV_IN_PROGRESS;
self->ith_msize = MACH_MSG_SIZE_MAX;
imq_unlock(reply_mqueue);
/*
* Finish extracting receiver from dest_mqueue.
*/
ipc_thread_rmqueue_first_macro(
&dest_mqueue->imq_threads, receiver);
imq_unlock(dest_mqueue);
/*
* Release the receiver's reference for his object.
*/
{
ipc_object_t object = receiver->ith_object;
io_lock(object);
io_release(object);
io_check_unlock(object);
}
{
struct mach_exception *exc =
(struct mach_exception *) &kmsg->ikm_header;
ipc_space_t space = receiver->task->itk_space;
/*
* We are running as the receiver now. We hold
* the following resources, which must be consumed:
* kmsg, send-once right for reply_port
* send rights for dest_port, thread_port, task_port
* Synthesize a kmsg for copyout to the receiver.
*/
exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PORT_SEND) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_size = sizeof *exc;
/* exc->Head.msgh_remote_port later */
/* exc->Head.msgh_local_port later */
exc->Head.msgh_seqno = 0;
exc->Head.msgh_id = MACH_EXCEPTION_ID;
exc->threadType = exc_port_proto;
/* exc->thread later */
exc->taskType = exc_port_proto;
/* exc->task later */
exc->exceptionType = exc_code_proto;
exc->exception = _exception;
exc->codeType = exc_code_proto;
exc->code = code;
exc->subcodeType = exc_code_proto;
exc->subcode = subcode;
/*
* Check that the receiver can handle the message.
*/
if (receiver->ith_rcv_size < sizeof(struct mach_exception)) {
/*
* ipc_kmsg_destroy is a handy way to consume
* the resources we hold, but it requires setup.
*/
exc->Head.msgh_bits =
(MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
exc->thread = (mach_port_t) thread_port;
exc->task = (mach_port_t) task_port;
ipc_kmsg_destroy(kmsg);
thread_syscall_return(MACH_RCV_TOO_LARGE);
/*NOTREACHED*/
}
is_write_lock(space);
assert(space->is_active);
/*
* To do an atomic copyout, need simultaneous
* locks on both ports and the space.
*/
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
abort_copyout:
ip_unlock(dest_port);
is_write_unlock(space);
/*
* Oh well, we have to do the header the slow way.
* First make it look like it's in-transit.
*/
exc->Head.msgh_bits =
(MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
mr = ipc_kmsg_copyout_header(&exc->Head, space,
MACH_PORT_NULL);
if (mr == MACH_MSG_SUCCESS)
goto copyout_body;
/*
* Ack! Prepare for ipc_kmsg_copyout_dest.
* It will consume thread_port and task_port.
*/
exc->thread = (mach_port_t) thread_port;
exc->task = (mach_port_t) task_port;
ipc_kmsg_copyout_dest(kmsg, space);
(void) ipc_kmsg_put(receiver->ith_msg, kmsg,
sizeof(mach_msg_header_t));
thread_syscall_return(mr);
/*NOTREACHED*/
}
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
goto abort_copyout;
}
assert(reply_port->ip_sorights > 0);
ip_unlock(reply_port);
{
kern_return_t kr;
ipc_entry_t entry;
kr = ipc_entry_get (space, &exc->Head.msgh_remote_port, &entry);
if (kr)
goto abort_copyout;
{
mach_port_gen_t gen;
assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
gen = entry->ie_bits + IE_BITS_GEN_ONE;
/* optimized ipc_right_copyout */
entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
}
entry->ie_object = (ipc_object_t) reply_port;
is_write_unlock(space);
}
/* optimized ipc_object_copyout_dest */
assert(dest_port->ip_srights > 0);
ip_release(dest_port);
exc->Head.msgh_local_port =
((dest_port->ip_receiver == space) ?
dest_port->ip_receiver_name : MACH_PORT_NULL);
if ((--dest_port->ip_srights == 0) &&
(dest_port->ip_nsrequest != IP_NULL)) {
ipc_port_t nsrequest;
mach_port_mscount_t mscount;
/* a rather rare case */
nsrequest = dest_port->ip_nsrequest;
mscount = dest_port->ip_mscount;
dest_port->ip_nsrequest = IP_NULL;
ip_unlock(dest_port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(dest_port);
copyout_body:
/*
* Optimized version of ipc_kmsg_copyout_body,
* to handle the two ports in the body.
*/
mr = (ipc_kmsg_copyout_object(space, (ipc_object_t) thread_port,
MACH_MSG_TYPE_PORT_SEND, &exc->thread) |
ipc_kmsg_copyout_object(space, (ipc_object_t) task_port,
MACH_MSG_TYPE_PORT_SEND, &exc->task));
if (mr != MACH_MSG_SUCCESS) {
(void) ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr | MACH_RCV_BODY_ERROR);
/*NOTREACHED*/
}
}
/*
* Optimized version of ipc_kmsg_put.
* We must check ikm_cache after copyoutmsg.
*/
ikm_check_initialized(kmsg, kmsg->ikm_size);
assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
if (copyoutmsg(&kmsg->ikm_header, receiver->ith_msg,
sizeof(struct mach_exception)) ||
(ikm_cache() != IKM_NULL)) {
mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
ikm_cache() = kmsg;
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
#ifndef __GNUC__
return; /* help for the compiler */
#endif
slow_exception_raise: {
struct mach_exception *exc =
(struct mach_exception *) &kmsg->ikm_header;
ipc_kmsg_t reply_kmsg;
mach_port_seqno_t reply_seqno;
exception_raise_misses++;
/*
* We hold the following resources, which must be consumed:
* kmsg, send-once right and ref for reply_port
* send rights for dest_port, thread_port, task_port
* Synthesize a kmsg to send.
*/
exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE) |
MACH_MSGH_BITS_COMPLEX);
exc->Head.msgh_size = sizeof *exc;
exc->Head.msgh_remote_port = (mach_port_t) dest_port;
exc->Head.msgh_local_port = (mach_port_t) reply_port;
exc->Head.msgh_seqno = 0;
exc->Head.msgh_id = MACH_EXCEPTION_ID;
exc->threadType = exc_port_proto;
exc->thread = (mach_port_t) thread_port;
exc->taskType = exc_port_proto;
exc->task = (mach_port_t) task_port;
exc->exceptionType = exc_code_proto;
exc->exception = _exception;
exc->codeType = exc_code_proto;
exc->code = code;
exc->subcodeType = exc_code_proto;
exc->subcode = subcode;
ipc_mqueue_send_always(kmsg);
/*
* We are left with a ref for reply_port,
* which we use to receive the reply message.
*/
ip_lock(reply_port);
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
exception_raise_continue_slow(MACH_RCV_PORT_DIED, IKM_NULL, /*dummy*/0);
/*NOTREACHED*/
}
imq_lock(reply_mqueue);
ip_unlock(reply_port);
mr = ipc_mqueue_receive(reply_mqueue, MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
FALSE, exception_raise_continue,
&reply_kmsg, &reply_seqno);
/* reply_mqueue is unlocked */
exception_raise_continue_slow(mr, reply_kmsg, reply_seqno);
/*NOTREACHED*/
}
}