ipc_port_t ipc_port_copy_send( ipc_port_t port) { ipc_port_t sright; if (!IP_VALID(port)) return port; ip_lock(port); if (ip_active(port)) { assert(port->ip_srights > 0); ip_reference(port); port->ip_srights++; sright = port; } else sright = IP_DEAD; ip_unlock(port); return sright; }
/* * Routine: convert_port_to_task_name * Purpose: * Convert from a port to a task name. * Doesn't consume the port ref; produces a task name ref, * which may be null. * Conditions: * Nothing locked. */ task_name_t convert_port_to_task_name( ipc_port_t port) { task_name_t task = TASK_NULL; if (IP_VALID(port)) { ip_lock(port); if ( ip_active(port) && (ip_kotype(port) == IKOT_TASK || ip_kotype(port) == IKOT_TASK_NAME)) { task = (task_name_t)port->ip_kobject; assert(task != TASK_NAME_NULL); task_reference_internal(task); } ip_unlock(port); } return (task); }
/* * Called with memory_object locked. Unlocks memory_object. */ ipc_port_t xmm_object_by_memory_object_remote( ipc_port_t memory_object) { kern_return_t kr; ipc_port_t xmm_object; #if DIPC node_name node; #endif /* DIPC */ assert(IP_WAS_REMOTE(memory_object)); #if DIPC node = DIPC_ORIGIN_NODE(memory_object); ip_unlock(memory_object); kr = r_xmm_remote_object_by_memory_object(dipc_host_priv_port(node), memory_object, &xmm_object); #else /* DIPC */ panic("xmm_object_by_memory_object_remote: no transport"); #endif /* DIPC */ assert(kr == KERN_SUCCESS); return xmm_object; }
kern_return_t mach_port_get_srights( ipc_space_t space, mach_port_t name, mach_port_rights_t *srightsp) { ipc_port_t port; kern_return_t kr; mach_port_rights_t srights; if (space == IS_NULL) return KERN_INVALID_TASK; kr = ipc_port_translate_receive(space, name, &port); if (kr != KERN_SUCCESS) return kr; /* port is locked and active */ srights = port->ip_srights; ip_unlock(port); *srightsp = srights; return KERN_SUCCESS; }
void ipc_object_copyin_from_kernel( ipc_object_t object, mach_msg_type_name_t msgt_name) { assert(IO_VALID(object)); switch (msgt_name) { case MACH_MSG_TYPE_MOVE_RECEIVE: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name != MACH_PORT_NULL); assert(port->ip_receiver == ipc_space_kernel); /* relevant part of ipc_port_clear_receiver */ ipc_port_set_mscount(port, 0); port->ip_receiver_name = MACH_PORT_NULL; port->ip_destination = IP_NULL; ipc_port_flag_protected_payload_clear(port); ip_unlock(port); break; } case MACH_MSG_TYPE_COPY_SEND: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); if (ip_active(port)) { assert(port->ip_srights > 0); port->ip_srights++; } ip_reference(port); ip_unlock(port); break; } case MACH_MSG_TYPE_MAKE_SEND: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name != MACH_PORT_NULL); assert(port->ip_receiver == ipc_space_kernel); ip_reference(port); port->ip_mscount++; port->ip_srights++; ip_unlock(port); break; } case MACH_MSG_TYPE_MOVE_SEND: /* move naked send right into the message */ break; case MACH_MSG_TYPE_MAKE_SEND_ONCE: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name != MACH_PORT_NULL); assert(port->ip_receiver == ipc_space_kernel); ip_reference(port); port->ip_sorights++; ip_unlock(port); break; } case MACH_MSG_TYPE_MOVE_SEND_ONCE: /* move naked send-once right into the message */ break; default: #if MACH_ASSERT assert(!"ipc_object_copyin_from_kernel: strange rights"); #else panic("ipc_object_copyin_from_kernel: strange rights"); #endif } }
/* * XXX who cares about send rights anymore XXX * Return send right for xmm object corresponding to memory object. * This is to be consumed when using xmm object to set up init, * either via move_send dest in proxy_init, or explicit deallocation * in local case. * Also returns one xmm_object ref, to be given to svm layer and * released there upon termination via xmm_object_release. * Also returns one xmm obj ref, to be consumed by xmm_obj_allocate * in either _proxy_init or xmm_memory_object_init. * * Create xmm object if necessary. * Memory object holds a send right to xmm object as well, which is released * when xmm object refs drop to 0. No-senders then triggers XXXXXX * svm deallocation. */ ipc_port_t xmm_object_by_memory_object( ipc_port_t memory_object) { ipc_port_t xmm_object, old_xmm_object; xmm_obj_t mobj; /* * We always create the svm stack at the current location of the * memory object. We may have to chase it down if it's migrating. * * The memory_object principal node is the one true source * of knowledge about whether an svm stack exists. */ ip_lock(memory_object); xmm_object_wait(memory_object); if (IP_WAS_REMOTE(memory_object)) { /* the following call inherits the lock */ return xmm_object_by_memory_object_remote(memory_object); } /* * If there is already an xmm_object associated with this * memory_object, return it, after taking a send-right reference * which will be given (moved, if necessary) to the caller. */ xmm_object = xmm_object_copy(memory_object); if (xmm_object != IP_NULL) { ip_unlock(memory_object); assert(!IP_IS_REMOTE(xmm_object)); return xmm_object; } /* * Check kobject type, to foil attempts to map in inappropriate * kernel objects (like task ports). */ if (ip_kotype(memory_object) != IKOT_NONE && ip_kotype(memory_object) != IKOT_PAGER) { ip_unlock(memory_object); return IP_NULL; } /* * No xmm object is currently associcated with memory object. * Unlock memory object port, and create an xmm obj stack. * and a corresponding xmm obj stack. * * XXX * Should deallocate things if this call fails part-way. */ ip_unlock(memory_object); xmm_user_create(memory_object, &mobj); xmm_svm_create(mobj, memory_object, &mobj); /* * Create an xmm object and associate it with stack. * It will have one send right. */ xmm_object = xmm_object_allocate(mobj); if (xmm_object == IP_NULL) { panic("xmm_mo_create: xmm_object_allocate"); return IP_NULL; } ip_lock(memory_object); assert(!IP_WAS_REMOTE(memory_object)); /* * If we lost the race to create the stack, discard ours * and use the one already created. Otherwise, associate * our xmm object and stack with the memory object, * by giving the memory object the send right to the xmm object. */ old_xmm_object = xmm_object_copy(memory_object); if (old_xmm_object != IP_NULL) { ip_unlock(memory_object); xmm_object_destroy(xmm_object, mobj); xmm_object = old_xmm_object; } else { xmm_object_set(memory_object, xmm_object, TRUE); ip_unlock(memory_object); } /* * Return the xmm object send right. */ return xmm_object; }
boolean_t ipc_right_copyin_check( ipc_space_t space, mach_port_t name, ipc_entry_t entry, mach_msg_type_name_t msgt_name) { ipc_entry_bits_t bits = entry->ie_bits; assert(space->is_active); switch (msgt_name) { case MACH_MSG_TYPE_MAKE_SEND: case MACH_MSG_TYPE_MAKE_SEND_ONCE: case MACH_MSG_TYPE_MOVE_RECEIVE: if ((bits & MACH_PORT_TYPE_RECEIVE) == 0) return FALSE; break; case MACH_MSG_TYPE_COPY_SEND: case MACH_MSG_TYPE_MOVE_SEND: case MACH_MSG_TYPE_MOVE_SEND_ONCE: { ipc_port_t port; boolean_t active; if (bits & MACH_PORT_TYPE_DEAD_NAME) break; if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0) return FALSE; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); ip_lock(port); active = ip_active(port); ip_unlock(port); if (!active) { break; } if (msgt_name == MACH_MSG_TYPE_MOVE_SEND_ONCE) { if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0) return FALSE; } else { if ((bits & MACH_PORT_TYPE_SEND) == 0) return FALSE; } break; } default: #if MACH_ASSERT assert(!"ipc_right_copyin_check: strange rights"); #else panic("ipc_right_copyin_check: strange rights"); #endif } return TRUE; }
void ipc_port_destroy( ipc_port_t port) { ipc_port_t pdrequest, nsrequest; ipc_mqueue_t mqueue; ipc_kmsg_t kmsg; #if IMPORTANCE_INHERITANCE task_t release_imp_task = TASK_NULL; thread_t self = current_thread(); boolean_t top = (self->ith_assertions == 0); natural_t assertcnt = 0; #endif /* IMPORTANCE_INHERITANCE */ assert(ip_active(port)); /* port->ip_receiver_name is garbage */ /* port->ip_receiver/port->ip_destination is garbage */ assert(port->ip_pset_count == 0); assert(port->ip_mscount == 0); /* check for a backup port */ pdrequest = port->ip_pdrequest; #if IMPORTANCE_INHERITANCE /* determine how may assertions to drop and from whom */ if (port->ip_tempowner != 0) { assert(top); if (port->ip_taskptr != 0) { release_imp_task = port->ip_imp_task; port->ip_imp_task = TASK_NULL; port->ip_taskptr = 0; assertcnt = port->ip_impcount; } /* Otherwise, nothing to drop */ } else { assert(port->ip_taskptr == 0); assertcnt = port->ip_impcount; if (pdrequest != IP_NULL) /* mark in limbo for the journey */ port->ip_tempowner = 1; } if (top) self->ith_assertions = assertcnt; #endif /* IMPORTANCE_INHERITANCE */ if (pdrequest != IP_NULL) { /* we assume the ref for pdrequest */ port->ip_pdrequest = IP_NULL; /* make port be in limbo */ port->ip_receiver_name = MACH_PORT_NULL; port->ip_destination = IP_NULL; ip_unlock(port); /* consumes our refs for port and pdrequest */ ipc_notify_port_destroyed(pdrequest, port); goto drop_assertions; } /* once port is dead, we don't need to keep it locked */ port->ip_object.io_bits &= ~IO_BITS_ACTIVE; port->ip_timestamp = ipc_port_timestamp(); /* * If the port has a preallocated message buffer and that buffer * is not inuse, free it. If it has an inuse one, then the kmsg * free will detect that we freed the association and it can free it * like a normal buffer. */ if (IP_PREALLOC(port)) { ipc_port_t inuse_port; kmsg = port->ip_premsg; assert(kmsg != IKM_NULL); inuse_port = ikm_prealloc_inuse_port(kmsg); IP_CLEAR_PREALLOC(port, kmsg); ip_unlock(port); if (inuse_port != IP_NULL) { assert(inuse_port == port); } else { ipc_kmsg_free(kmsg); } } else { ip_unlock(port); } /* throw away no-senders request */ nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) ipc_notify_send_once(nsrequest); /* consumes ref */ /* destroy any queued messages */ mqueue = &port->ip_messages; ipc_mqueue_destroy(mqueue); /* generate dead-name notifications */ ipc_port_dnnotify(port); ipc_kobject_destroy(port); ip_release(port); /* consume caller's ref */ drop_assertions: #if IMPORTANCE_INHERITANCE if (release_imp_task != TASK_NULL) { if (assertcnt > 0) { assert(top); self->ith_assertions = 0; assert(release_imp_task->imp_receiver != 0); task_importance_drop_internal_assertion(release_imp_task, assertcnt); } task_deallocate(release_imp_task); } else if (assertcnt > 0) { if (top) { self->ith_assertions = 0; release_imp_task = current_task(); if (release_imp_task->imp_receiver != 0) { task_importance_drop_internal_assertion(release_imp_task, assertcnt); } } else { /* the port chain we are enqueued on should cover our assertions */ assert(assertcnt <= self->ith_assertions); } } #endif /* IMPORTANCE_INHERITANCE */ }
kern_return_t ipc_port_request_grow( ipc_port_t port, ipc_table_elems_t target_size) { ipc_table_size_t its; ipc_port_request_t otable, ntable; assert(ip_active(port)); otable = port->ip_requests; if (otable == IPR_NULL) its = &ipc_table_requests[0]; else its = otable->ipr_size + 1; if (target_size != ITS_SIZE_NONE) { if ((otable != IPR_NULL) && (target_size <= otable->ipr_size->its_size)) { ip_unlock(port); return KERN_SUCCESS; } while ((its->its_size) && (its->its_size < target_size)) { its++; } if (its->its_size == 0) { ip_unlock(port); return KERN_NO_SPACE; } } ip_reference(port); ip_unlock(port); if ((its->its_size == 0) || ((ntable = it_requests_alloc(its)) == IPR_NULL)) { ip_release(port); return KERN_RESOURCE_SHORTAGE; } ip_lock(port); /* * Check that port is still active and that nobody else * has slipped in and grown the table on us. Note that * just checking if the current table pointer == otable * isn't sufficient; must check ipr_size. */ if (ip_active(port) && (port->ip_requests == otable) && ((otable == IPR_NULL) || (otable->ipr_size+1 == its))) { ipc_table_size_t oits; ipc_table_elems_t osize, nsize; ipc_port_request_index_t free, i; /* copy old table to new table */ if (otable != IPR_NULL) { oits = otable->ipr_size; osize = oits->its_size; free = otable->ipr_next; (void) memcpy((void *)(ntable + 1), (const void *)(otable + 1), (osize - 1) * sizeof(struct ipc_port_request)); } else { osize = 1; oits = 0; free = 0; } nsize = its->its_size; assert(nsize > osize); /* add new elements to the new table's free list */ for (i = osize; i < nsize; i++) { ipc_port_request_t ipr = &ntable[i]; ipr->ipr_name = MACH_PORT_NULL; ipr->ipr_next = free; free = i; } ntable->ipr_next = free; ntable->ipr_size = its; port->ip_requests = ntable; ip_unlock(port); ip_release(port); if (otable != IPR_NULL) { it_requests_free(oits, otable); } } else { ip_unlock(port); ip_release(port); it_requests_free(its, ntable); } return KERN_SUCCESS; }
kern_return_t ipc_right_dnrequest( ipc_space_t space, mach_port_t name, boolean_t immediate, ipc_port_t notify, ipc_port_t *previousp) { ipc_port_t previous; for (;;) { ipc_entry_t entry; ipc_entry_bits_t bits; kern_return_t kr; kr = ipc_right_lookup_write(space, name, &entry); if (kr != KERN_SUCCESS) return kr; /* space is write-locked and active */ bits = entry->ie_bits; if (bits & MACH_PORT_TYPE_PORT_RIGHTS) { ipc_port_t port; ipc_port_request_index_t request; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (!ipc_right_check(space, port, name, entry)) { /* port is locked and active */ if (notify == IP_NULL) { previous = ipc_right_dncancel_macro( space, port, name, entry); ip_unlock(port); is_write_unlock(space); break; } /* * If a registered soright exists, * want to atomically switch with it. * If ipc_port_dncancel finds us a * soright, then the following * ipc_port_dnrequest will reuse * that slot, so we are guaranteed * not to unlock and retry. */ previous = ipc_right_dncancel_macro(space, port, name, entry); kr = ipc_port_dnrequest(port, name, notify, &request); if (kr != KERN_SUCCESS) { assert(previous == IP_NULL); is_write_unlock(space); kr = ipc_port_dngrow(port); /* port is unlocked */ if (kr != KERN_SUCCESS) return kr; continue; } assert(request != 0); ip_unlock(port); entry->ie_request = request; is_write_unlock(space); break; } bits = entry->ie_bits; assert(bits & MACH_PORT_TYPE_DEAD_NAME); } if ((bits & MACH_PORT_TYPE_DEAD_NAME) && immediate && (notify != IP_NULL)) { mach_port_urefs_t urefs = IE_BITS_UREFS(bits); assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME); assert(urefs > 0); if (MACH_PORT_UREFS_OVERFLOW(urefs, 1)) { is_write_unlock(space); return KERN_UREFS_OVERFLOW; } entry->ie_bits = bits + 1; /* increment urefs */ is_write_unlock(space); ipc_notify_dead_name(notify, name); previous = IP_NULL; break; } is_write_unlock(space); if (bits & MACH_PORT_TYPE_PORT_OR_DEAD) return KERN_INVALID_ARGUMENT; else return KERN_INVALID_RIGHT; } *previousp = previous; return KERN_SUCCESS; }
kern_return_t ipc_right_copyin( ipc_space_t space, mach_port_t name, ipc_entry_t entry, mach_msg_type_name_t msgt_name, boolean_t deadok, ipc_object_t *objectp, ipc_port_t *sorightp) { ipc_entry_bits_t bits = entry->ie_bits; assert(space->is_active); switch (msgt_name) { case MACH_MSG_TYPE_MAKE_SEND: { ipc_port_t port; if ((bits & MACH_PORT_TYPE_RECEIVE) == 0) goto invalid_right; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); port->ip_mscount++; port->ip_srights++; ip_reference(port); ip_unlock(port); *objectp = (ipc_object_t) port; *sorightp = IP_NULL; break; } case MACH_MSG_TYPE_MAKE_SEND_ONCE: { ipc_port_t port; if ((bits & MACH_PORT_TYPE_RECEIVE) == 0) goto invalid_right; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); port->ip_sorights++; ip_reference(port); ip_unlock(port); *objectp = (ipc_object_t) port; *sorightp = IP_NULL; break; } case MACH_MSG_TYPE_MOVE_RECEIVE: { ipc_port_t port; ipc_port_t dnrequest = IP_NULL; if ((bits & MACH_PORT_TYPE_RECEIVE) == 0) goto invalid_right; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); if (bits & MACH_PORT_TYPE_SEND) { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_RECEIVE); assert(IE_BITS_UREFS(bits) > 0); assert(port->ip_srights > 0); entry->ie_name = name; ipc_reverse_insert(space, (ipc_object_t) port, entry); ip_reference(port); } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE); assert(IE_BITS_UREFS(bits) == 0); dnrequest = ipc_right_dncancel_macro(space, port, name, entry); if (bits & IE_BITS_MAREQUEST) ipc_marequest_cancel(space, name); entry->ie_object = IO_NULL; } entry->ie_bits = bits &~ MACH_PORT_TYPE_RECEIVE; ipc_port_clear_receiver(port); port->ip_receiver_name = MACH_PORT_NULL; port->ip_destination = IP_NULL; /* * Clear the protected payload field to retain * the behavior of mach_msg. */ ipc_port_flag_protected_payload_clear(port); ip_unlock(port); *objectp = (ipc_object_t) port; *sorightp = dnrequest; break; } case MACH_MSG_TYPE_COPY_SEND: { ipc_port_t port; if (bits & MACH_PORT_TYPE_DEAD_NAME) goto copy_dead; /* allow for dead send-once rights */ if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0) goto invalid_right; assert(IE_BITS_UREFS(bits) > 0); port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { bits = entry->ie_bits; goto copy_dead; } /* port is locked and active */ if ((bits & MACH_PORT_TYPE_SEND) == 0) { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE); assert(port->ip_sorights > 0); ip_unlock(port); goto invalid_right; } assert(port->ip_srights > 0); port->ip_srights++; ip_reference(port); ip_unlock(port); *objectp = (ipc_object_t) port; *sorightp = IP_NULL; break; } case MACH_MSG_TYPE_MOVE_SEND: { ipc_port_t port; ipc_port_t dnrequest = IP_NULL; if (bits & MACH_PORT_TYPE_DEAD_NAME) goto move_dead; /* allow for dead send-once rights */ if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0) goto invalid_right; assert(IE_BITS_UREFS(bits) > 0); port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { bits = entry->ie_bits; goto move_dead; } /* port is locked and active */ if ((bits & MACH_PORT_TYPE_SEND) == 0) { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE); assert(port->ip_sorights > 0); ip_unlock(port); goto invalid_right; } assert(port->ip_srights > 0); if (IE_BITS_UREFS(bits) == 1) { if (bits & MACH_PORT_TYPE_RECEIVE) { assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_RECEIVE); ip_reference(port); } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND); dnrequest = ipc_right_dncancel_macro( space, port, name, entry); ipc_reverse_remove(space, (ipc_object_t) port); if (bits & IE_BITS_MAREQUEST) ipc_marequest_cancel(space, name); entry->ie_object = IO_NULL; } entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK|MACH_PORT_TYPE_SEND); } else { port->ip_srights++; ip_reference(port); entry->ie_bits = bits-1; /* decrement urefs */ } ip_unlock(port); *objectp = (ipc_object_t) port; *sorightp = dnrequest; break; } case MACH_MSG_TYPE_MOVE_SEND_ONCE: { ipc_port_t port; ipc_port_t dnrequest; if (bits & MACH_PORT_TYPE_DEAD_NAME) goto move_dead; /* allow for dead send rights */ if ((bits & MACH_PORT_TYPE_SEND_RIGHTS) == 0) goto invalid_right; assert(IE_BITS_UREFS(bits) > 0); port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { bits = entry->ie_bits; goto move_dead; } /* port is locked and active */ if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0) { assert(bits & MACH_PORT_TYPE_SEND); assert(port->ip_srights > 0); ip_unlock(port); goto invalid_right; } assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE); assert(IE_BITS_UREFS(bits) == 1); assert((bits & IE_BITS_MAREQUEST) == 0); assert(port->ip_sorights > 0); dnrequest = ipc_right_dncancel_macro(space, port, name, entry); ip_unlock(port); entry->ie_object = IO_NULL; entry->ie_bits = bits &~ MACH_PORT_TYPE_SEND_ONCE; *objectp = (ipc_object_t) port; *sorightp = dnrequest; break; } default: #if MACH_ASSERT assert(!"ipc_right_copyin: strange rights"); #else panic("ipc_right_copyin: strange rights"); #endif } return KERN_SUCCESS; copy_dead: assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME); assert(IE_BITS_UREFS(bits) > 0); assert((bits & IE_BITS_MAREQUEST) == 0); assert(entry->ie_request == 0); assert(entry->ie_object == 0); if (!deadok) goto invalid_right; *objectp = IO_DEAD; *sorightp = IP_NULL; return KERN_SUCCESS; move_dead: assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME); assert(IE_BITS_UREFS(bits) > 0); assert((bits & IE_BITS_MAREQUEST) == 0); assert(entry->ie_request == 0); assert(entry->ie_object == 0); if (!deadok) goto invalid_right; if (IE_BITS_UREFS(bits) == 1) entry->ie_bits = bits &~ MACH_PORT_TYPE_DEAD_NAME; else entry->ie_bits = bits-1; /* decrement urefs */ *objectp = IO_DEAD; *sorightp = IP_NULL; return KERN_SUCCESS; invalid_right: return KERN_INVALID_RIGHT; }
kern_return_t ipc_right_dealloc( ipc_space_t space, mach_port_t name, ipc_entry_t entry) { ipc_entry_bits_t bits = entry->ie_bits; mach_port_type_t type = IE_BITS_TYPE(bits); assert(space->is_active); switch (type) { case MACH_PORT_TYPE_DEAD_NAME: { dead_name: assert(IE_BITS_UREFS(bits) > 0); assert(entry->ie_request == 0); assert(entry->ie_object == IO_NULL); assert((bits & IE_BITS_MAREQUEST) == 0); if (IE_BITS_UREFS(bits) == 1) ipc_entry_dealloc(space, name, entry); else entry->ie_bits = bits-1; /* decrement urefs */ is_write_unlock(space); break; } case MACH_PORT_TYPE_SEND_ONCE: { ipc_port_t port, dnrequest; assert(IE_BITS_UREFS(bits) == 1); assert((bits & IE_BITS_MAREQUEST) == 0); port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { bits = entry->ie_bits; assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME); goto dead_name; } /* port is locked and active */ assert(port->ip_sorights > 0); dnrequest = ipc_right_dncancel_macro(space, port, name, entry); ip_unlock(port); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); is_write_unlock(space); ipc_notify_send_once(port); if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } case MACH_PORT_TYPE_SEND: { ipc_port_t port; ipc_port_t dnrequest = IP_NULL; ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */ assert(IE_BITS_UREFS(bits) > 0); port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { bits = entry->ie_bits; assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME); goto dead_name; } /* port is locked and active */ assert(port->ip_srights > 0); if (IE_BITS_UREFS(bits) == 1) { if (--port->ip_srights == 0) { nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) { port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; } } dnrequest = ipc_right_dncancel_macro(space, port, name, entry); ipc_reverse_remove(space, (ipc_object_t) port); if (bits & IE_BITS_MAREQUEST) ipc_marequest_cancel(space, name); ip_release(port); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); } else entry->ie_bits = bits-1; /* decrement urefs */ ip_unlock(port); /* even if dropped a ref, port is active */ is_write_unlock(space); if (nsrequest != IP_NULL) ipc_notify_no_senders(nsrequest, mscount); if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } case MACH_PORT_TYPE_SEND_RECEIVE: { ipc_port_t port; ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */ assert(IE_BITS_UREFS(bits) > 0); port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); assert(port->ip_srights > 0); if (IE_BITS_UREFS(bits) == 1) { if (--port->ip_srights == 0) { nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) { port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; } } entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK| MACH_PORT_TYPE_SEND); } else entry->ie_bits = bits-1; /* decrement urefs */ ip_unlock(port); is_write_unlock(space); if (nsrequest != IP_NULL) ipc_notify_no_senders(nsrequest, mscount); break; } default: is_write_unlock(space); return KERN_INVALID_RIGHT; } return KERN_SUCCESS; }
kern_return_t ipc_right_delta( ipc_space_t space, mach_port_t name, ipc_entry_t entry, mach_port_right_t right, mach_port_delta_t delta) { ipc_entry_bits_t bits = entry->ie_bits; assert(space->is_active); assert(right < MACH_PORT_RIGHT_NUMBER); /* Rights-specific restrictions and operations. */ switch (right) { case MACH_PORT_RIGHT_PORT_SET: { ipc_pset_t pset; if ((bits & MACH_PORT_TYPE_PORT_SET) == 0) goto invalid_right; assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_PORT_SET); assert(IE_BITS_UREFS(bits) == 0); assert((bits & IE_BITS_MAREQUEST) == 0); assert(entry->ie_request == 0); if (delta == 0) goto success; if (delta != -1) goto invalid_value; pset = (ipc_pset_t) entry->ie_object; assert(pset != IPS_NULL); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); ips_lock(pset); assert(ips_active(pset)); is_write_unlock(space); ipc_pset_destroy(pset); /* consumes ref, unlocks */ break; } case MACH_PORT_RIGHT_RECEIVE: { ipc_port_t port; ipc_port_t dnrequest = IP_NULL; if ((bits & MACH_PORT_TYPE_RECEIVE) == 0) goto invalid_right; if (delta == 0) goto success; if (delta != -1) goto invalid_value; if (bits & IE_BITS_MAREQUEST) { bits &= ~IE_BITS_MAREQUEST; ipc_marequest_cancel(space, name); } port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); /* * The port lock is needed for ipc_right_dncancel; * otherwise, we wouldn't have to take the lock * until just before dropping the space lock. */ ip_lock(port); assert(ip_active(port)); assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); if (bits & MACH_PORT_TYPE_SEND) { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_RECEIVE); assert(IE_BITS_UREFS(bits) > 0); assert(IE_BITS_UREFS(bits) < MACH_PORT_UREFS_MAX); assert(port->ip_srights > 0); /* * The remaining send right turns into a * dead name. Notice we don't decrement * ip_srights, generate a no-senders notif, * or use ipc_right_dncancel, because the * port is destroyed "first". */ bits &= ~IE_BITS_TYPE_MASK; bits |= MACH_PORT_TYPE_DEAD_NAME; if (entry->ie_request != 0) { entry->ie_request = 0; bits++; /* increment urefs */ } entry->ie_bits = bits; entry->ie_object = IO_NULL; } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE); assert(IE_BITS_UREFS(bits) == 0); dnrequest = ipc_right_dncancel_macro(space, port, name, entry); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); } is_write_unlock(space); ipc_port_clear_receiver(port); ipc_port_destroy(port); /* consumes ref, unlocks */ if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } case MACH_PORT_RIGHT_SEND_ONCE: { ipc_port_t port, dnrequest; if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0) goto invalid_right; assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE); assert(IE_BITS_UREFS(bits) == 1); assert((bits & IE_BITS_MAREQUEST) == 0); if ((delta > 0) || (delta < -1)) goto invalid_value; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { assert(!(entry->ie_bits & MACH_PORT_TYPE_SEND_ONCE)); goto invalid_right; } /* port is locked and active */ assert(port->ip_sorights > 0); if (delta == 0) { ip_unlock(port); goto success; } dnrequest = ipc_right_dncancel_macro(space, port, name, entry); ip_unlock(port); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); is_write_unlock(space); ipc_notify_send_once(port); if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } case MACH_PORT_RIGHT_DEAD_NAME: { mach_port_urefs_t urefs; if (bits & MACH_PORT_TYPE_SEND_RIGHTS) { ipc_port_t port; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (!ipc_right_check(space, port, name, entry)) { /* port is locked and active */ ip_unlock(port); goto invalid_right; } bits = entry->ie_bits; } else if ((bits & MACH_PORT_TYPE_DEAD_NAME) == 0) goto invalid_right; assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_DEAD_NAME); assert(IE_BITS_UREFS(bits) > 0); assert((bits & IE_BITS_MAREQUEST) == 0); assert(entry->ie_object == IO_NULL); assert(entry->ie_request == 0); urefs = IE_BITS_UREFS(bits); if (MACH_PORT_UREFS_UNDERFLOW(urefs, delta)) goto invalid_value; if (MACH_PORT_UREFS_OVERFLOW(urefs, delta)) goto urefs_overflow; if ((urefs + delta) == 0) ipc_entry_dealloc(space, name, entry); else entry->ie_bits = bits + delta; is_write_unlock(space); break; } case MACH_PORT_RIGHT_SEND: { mach_port_urefs_t urefs; ipc_port_t port; ipc_port_t dnrequest = IP_NULL; ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */ if ((bits & MACH_PORT_TYPE_SEND) == 0) goto invalid_right; /* maximum urefs for send is MACH_PORT_UREFS_MAX-1 */ urefs = IE_BITS_UREFS(bits); if (MACH_PORT_UREFS_UNDERFLOW(urefs, delta)) goto invalid_value; if (MACH_PORT_UREFS_OVERFLOW(urefs+1, delta)) goto urefs_overflow; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { assert((entry->ie_bits & MACH_PORT_TYPE_SEND) == 0); goto invalid_right; } /* port is locked and active */ assert(port->ip_srights > 0); if ((urefs + delta) == 0) { if (--port->ip_srights == 0) { nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) { port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; } } if (bits & MACH_PORT_TYPE_RECEIVE) { assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_RECEIVE); entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK| MACH_PORT_TYPE_SEND); } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND); dnrequest = ipc_right_dncancel_macro( space, port, name, entry); ipc_reverse_remove(space, (ipc_object_t) port); if (bits & IE_BITS_MAREQUEST) ipc_marequest_cancel(space, name); ip_release(port); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); } } else entry->ie_bits = bits + delta; ip_unlock(port); /* even if dropped a ref, port is active */ is_write_unlock(space); if (nsrequest != IP_NULL) ipc_notify_no_senders(nsrequest, mscount); if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } default: #if MACH_ASSERT assert(!"ipc_right_delta: strange right"); #else panic("ipc_right_delta: strange right"); #endif } return KERN_SUCCESS; success: is_write_unlock(space); return KERN_SUCCESS; invalid_right: is_write_unlock(space); return KERN_INVALID_RIGHT; invalid_value: is_write_unlock(space); return KERN_INVALID_VALUE; urefs_overflow: is_write_unlock(space); return KERN_UREFS_OVERFLOW; }
kern_return_t ipc_right_destroy( ipc_space_t space, mach_port_t name, ipc_entry_t entry) { ipc_entry_bits_t bits = entry->ie_bits; mach_port_type_t type = IE_BITS_TYPE(bits); assert(space->is_active); switch (type) { case MACH_PORT_TYPE_DEAD_NAME: assert(entry->ie_request == 0); assert(entry->ie_object == IO_NULL); assert((bits & IE_BITS_MAREQUEST) == 0); ipc_entry_dealloc(space, name, entry); is_write_unlock(space); break; case MACH_PORT_TYPE_PORT_SET: { ipc_pset_t pset = (ipc_pset_t) entry->ie_object; assert(entry->ie_request == 0); assert(pset != IPS_NULL); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); ips_lock(pset); assert(ips_active(pset)); is_write_unlock(space); ipc_pset_destroy(pset); /* consumes ref, unlocks */ break; } case MACH_PORT_TYPE_SEND: case MACH_PORT_TYPE_RECEIVE: case MACH_PORT_TYPE_SEND_RECEIVE: case MACH_PORT_TYPE_SEND_ONCE: { ipc_port_t port = (ipc_port_t) entry->ie_object; ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */ ipc_port_t dnrequest; assert(port != IP_NULL); if (bits & IE_BITS_MAREQUEST) { assert(type & MACH_PORT_TYPE_SEND_RECEIVE); ipc_marequest_cancel(space, name); } if (type == MACH_PORT_TYPE_SEND) ipc_reverse_remove(space, (ipc_object_t) port); ip_lock(port); if (!ip_active(port)) { assert((type & MACH_PORT_TYPE_RECEIVE) == 0); ip_release(port); ip_check_unlock(port); entry->ie_request = 0; entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); is_write_unlock(space); break; } dnrequest = ipc_right_dncancel_macro(space, port, name, entry); entry->ie_object = IO_NULL; ipc_entry_dealloc(space, name, entry); is_write_unlock(space); if (type & MACH_PORT_TYPE_SEND) { assert(port->ip_srights > 0); if (--port->ip_srights == 0) { nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) { port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; } } } if (type & MACH_PORT_TYPE_RECEIVE) { assert(ip_active(port)); assert(port->ip_receiver == space); ipc_port_clear_receiver(port); ipc_port_destroy(port); /* consumes our ref, unlocks */ } else if (type & MACH_PORT_TYPE_SEND_ONCE) { assert(port->ip_sorights > 0); ip_unlock(port); ipc_notify_send_once(port); /* consumes our ref */ } else { assert(port->ip_receiver != space); ip_release(port); ip_unlock(port); } if (nsrequest != IP_NULL) ipc_notify_no_senders(nsrequest, mscount); if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } default: #if MACH_ASSERT assert(!"ipc_right_destroy: strange type"); #else panic("ipc_right_destroy: strange type"); #endif } return KERN_SUCCESS; }
void ipc_right_clean( ipc_space_t space, mach_port_t name, ipc_entry_t entry) { ipc_entry_bits_t bits = entry->ie_bits; mach_port_type_t type = IE_BITS_TYPE(bits); assert(!space->is_active); /* * We can't clean up IE_BITS_MAREQUEST when the space is dead. * This is because ipc_marequest_destroy can't turn off * the bit if the space is dead. Hence, it might be on * even though the marequest has been destroyed. It's OK * not to cancel the marequest, because ipc_marequest_destroy * cancels for us if the space is dead. * * IE_BITS_COMPAT/ipc_right_dncancel doesn't have this * problem, because we check that the port is active. If * we didn't cancel IE_BITS_COMPAT, ipc_port_destroy * would still work, but dead space refs would accumulate * in ip_dnrequests. They would use up slots in * ip_dnrequests and keep the spaces from being freed. */ switch (type) { case MACH_PORT_TYPE_DEAD_NAME: assert(entry->ie_request == 0); assert(entry->ie_object == IO_NULL); assert((bits & IE_BITS_MAREQUEST) == 0); break; case MACH_PORT_TYPE_PORT_SET: { ipc_pset_t pset = (ipc_pset_t) entry->ie_object; assert(entry->ie_request == 0); assert((bits & IE_BITS_MAREQUEST) == 0); assert(pset != IPS_NULL); ips_lock(pset); assert(ips_active(pset)); ipc_pset_destroy(pset); /* consumes ref, unlocks */ break; } case MACH_PORT_TYPE_SEND: case MACH_PORT_TYPE_RECEIVE: case MACH_PORT_TYPE_SEND_RECEIVE: case MACH_PORT_TYPE_SEND_ONCE: { ipc_port_t port = (ipc_port_t) entry->ie_object; ipc_port_t dnrequest; ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount = 0; /* '=0' to shut up lint */ assert(port != IP_NULL); ip_lock(port); if (!ip_active(port)) { ip_release(port); ip_check_unlock(port); break; } dnrequest = ipc_right_dncancel_macro(space, port, name, entry); if (type & MACH_PORT_TYPE_SEND) { assert(port->ip_srights > 0); if (--port->ip_srights == 0) { nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) { port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; } } } if (type & MACH_PORT_TYPE_RECEIVE) { assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); ipc_port_clear_receiver(port); ipc_port_destroy(port); /* consumes our ref, unlocks */ } else if (type & MACH_PORT_TYPE_SEND_ONCE) { assert(port->ip_sorights > 0); ip_unlock(port); ipc_notify_send_once(port); /* consumes our ref */ } else { assert(port->ip_receiver != space); ip_release(port); ip_unlock(port); /* port is active */ } if (nsrequest != IP_NULL) ipc_notify_no_senders(nsrequest, mscount); if (dnrequest != IP_NULL) ipc_notify_port_deleted(dnrequest, name); break; } default: #if MACH_ASSERT assert(!"ipc_right_clean: strange type"); #else panic("ipc_right_clean: strange type"); #endif } }
kern_return_t ipc_right_copyout( ipc_space_t space, mach_port_t name, ipc_entry_t entry, mach_msg_type_name_t msgt_name, boolean_t overflow, ipc_object_t object) { ipc_entry_bits_t bits = entry->ie_bits; ipc_port_t port; assert(IO_VALID(object)); assert(io_otype(object) == IOT_PORT); assert(io_active(object)); assert(entry->ie_object == object); port = (ipc_port_t) object; switch (msgt_name) { case MACH_MSG_TYPE_PORT_SEND_ONCE: assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE); assert(port->ip_sorights > 0); /* transfer send-once right and ref to entry */ ip_unlock(port); entry->ie_bits = bits | (MACH_PORT_TYPE_SEND_ONCE | 1); break; case MACH_MSG_TYPE_PORT_SEND: assert(port->ip_srights > 0); if (bits & MACH_PORT_TYPE_SEND) { mach_port_urefs_t urefs = IE_BITS_UREFS(bits); assert(port->ip_srights > 1); assert(urefs > 0); assert(urefs < MACH_PORT_UREFS_MAX); if (urefs+1 == MACH_PORT_UREFS_MAX) { if (overflow) { /* leave urefs pegged to maximum */ port->ip_srights--; ip_release(port); ip_unlock(port); return KERN_SUCCESS; } ip_unlock(port); return KERN_UREFS_OVERFLOW; } port->ip_srights--; ip_release(port); ip_unlock(port); } else if (bits & MACH_PORT_TYPE_RECEIVE) { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_RECEIVE); assert(IE_BITS_UREFS(bits) == 0); /* transfer send right to entry */ ip_release(port); ip_unlock(port); } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE); assert(IE_BITS_UREFS(bits) == 0); /* transfer send right and ref to entry */ ip_unlock(port); /* entry is locked holding ref, so can use port */ entry->ie_name = name; ipc_reverse_insert(space, (ipc_object_t) port, entry); } entry->ie_bits = (bits | MACH_PORT_TYPE_SEND) + 1; break; case MACH_MSG_TYPE_PORT_RECEIVE: { ipc_port_t dest; assert(port->ip_mscount == 0); assert(port->ip_receiver_name == MACH_PORT_NULL); dest = port->ip_destination; port->ip_receiver_name = name; port->ip_receiver = space; /* * Clear the protected payload field to retain * the behavior of mach_msg. */ ipc_port_flag_protected_payload_clear(port); assert((bits & MACH_PORT_TYPE_RECEIVE) == 0); if (bits & MACH_PORT_TYPE_SEND) { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND); assert(IE_BITS_UREFS(bits) > 0); assert(port->ip_srights > 0); ip_release(port); ip_unlock(port); /* entry is locked holding ref, so can use port */ ipc_reverse_remove(space, (ipc_object_t) port); } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE); assert(IE_BITS_UREFS(bits) == 0); /* transfer ref to entry */ ip_unlock(port); } entry->ie_bits = bits | MACH_PORT_TYPE_RECEIVE; if (dest != IP_NULL) ipc_port_release(dest); break; } default: #if MACH_ASSERT assert(!"ipc_right_copyout: strange rights"); #else panic("ipc_right_copyout: strange rights"); #endif } return KERN_SUCCESS; }
void ipc_port_destroy( ipc_port_t port) { ipc_port_t pdrequest, nsrequest; ipc_mqueue_t mqueue; ipc_kmsg_t kmsg; ipc_port_request_t dnrequests; assert(ip_active(port)); /* port->ip_receiver_name is garbage */ /* port->ip_receiver/port->ip_destination is garbage */ assert(port->ip_pset_count == 0); assert(port->ip_mscount == 0); /* first check for a backup port */ pdrequest = port->ip_pdrequest; if (pdrequest != IP_NULL) { /* we assume the ref for pdrequest */ port->ip_pdrequest = IP_NULL; /* make port be in limbo */ port->ip_receiver_name = MACH_PORT_NULL; port->ip_destination = IP_NULL; ip_unlock(port); /* consumes our refs for port and pdrequest */ ipc_notify_port_destroyed(pdrequest, port); return; } /* once port is dead, we don't need to keep it locked */ port->ip_object.io_bits &= ~IO_BITS_ACTIVE; port->ip_timestamp = ipc_port_timestamp(); /* save for later */ dnrequests = port->ip_dnrequests; port->ip_dnrequests = IPR_NULL; /* * If the port has a preallocated message buffer and that buffer * is not inuse, free it. If it has an inuse one, then the kmsg * free will detect that we freed the association and it can free it * like a normal buffer. */ if (IP_PREALLOC(port)) { kmsg = port->ip_premsg; assert(kmsg != IKM_NULL); IP_CLEAR_PREALLOC(port, kmsg); if (!ikm_prealloc_inuse(kmsg)) ipc_kmsg_free(kmsg); } ip_unlock(port); /* throw away no-senders request */ nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) ipc_notify_send_once(nsrequest); /* consumes ref */ /* destroy any queued messages */ mqueue = &port->ip_messages; ipc_mqueue_destroy(mqueue); /* generate dead-name notifications */ if (dnrequests != IPR_NULL) { ipc_port_dnnotify(port, dnrequests); } ipc_kobject_destroy(port); ipc_port_release(port); /* consume caller's ref */ }
void ipc_object_copyin_from_kernel( ipc_object_t object, mach_msg_type_name_t msgt_name) { assert(IO_VALID(object)); switch (msgt_name) { case MACH_MSG_TYPE_MOVE_RECEIVE: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); assert(ip_active(port)); if (port->ip_destination != IP_NULL) { assert(port->ip_receiver == ipc_space_kernel); /* relevant part of ipc_port_clear_receiver */ ipc_port_set_mscount(port, 0); port->ip_receiver_name = MACH_PORT_NULL; port->ip_destination = IP_NULL; } ip_unlock(port); break; } case MACH_MSG_TYPE_COPY_SEND: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); if (ip_active(port)) { assert(port->ip_srights > 0); port->ip_srights++; } ip_reference(port); ip_unlock(port); break; } case MACH_MSG_TYPE_MAKE_SEND: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); if (ip_active(port)) { assert(port->ip_receiver_name != MACH_PORT_NULL); assert((port->ip_receiver == ipc_space_kernel) || (port->ip_receiver->is_node_id != HOST_LOCAL_NODE)); port->ip_mscount++; } port->ip_srights++; ip_reference(port); ip_unlock(port); break; } case MACH_MSG_TYPE_MOVE_SEND: { /* move naked send right into the message */ assert(((ipc_port_t)object)->ip_srights); break; } case MACH_MSG_TYPE_MAKE_SEND_ONCE: { ipc_port_t port = (ipc_port_t) object; ip_lock(port); if (ip_active(port)) { assert(port->ip_receiver_name != MACH_PORT_NULL); } port->ip_sorights++; ip_reference(port); ip_unlock(port); break; } case MACH_MSG_TYPE_MOVE_SEND_ONCE: { /* move naked send-once right into the message */ assert(((ipc_port_t)object)->ip_sorights); break; } default: panic("ipc_object_copyin_from_kernel: strange rights"); } }
boolean_t ipc_port_check_circularity( ipc_port_t port, ipc_port_t dest) { ipc_port_t base; assert(port != IP_NULL); assert(dest != IP_NULL); if (port == dest) return TRUE; base = dest; /* * First try a quick check that can run in parallel. * No circularity if dest is not in transit. */ ip_lock(port); if (ip_lock_try(dest)) { if (!ip_active(dest) || (dest->ip_receiver_name != MACH_PORT_NULL) || (dest->ip_destination == IP_NULL)) goto not_circular; /* dest is in transit; further checking necessary */ ip_unlock(dest); } ip_unlock(port); ipc_port_multiple_lock(); /* massive serialization */ /* * Search for the end of the chain (a port not in transit), * acquiring locks along the way. */ for (;;) { ip_lock(base); if (!ip_active(base) || (base->ip_receiver_name != MACH_PORT_NULL) || (base->ip_destination == IP_NULL)) break; base = base->ip_destination; } /* all ports in chain from dest to base, inclusive, are locked */ if (port == base) { /* circularity detected! */ ipc_port_multiple_unlock(); /* port (== base) is in limbo */ assert(ip_active(port)); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination == IP_NULL); while (dest != IP_NULL) { ipc_port_t next; /* dest is in transit or in limbo */ assert(ip_active(dest)); assert(dest->ip_receiver_name == MACH_PORT_NULL); next = dest->ip_destination; ip_unlock(dest); dest = next; } return TRUE; } /* * The guarantee: lock port while the entire chain is locked. * Once port is locked, we can take a reference to dest, * add port to the chain, and unlock everything. */ ip_lock(port); ipc_port_multiple_unlock(); not_circular: /* port is in limbo */ assert(ip_active(port)); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination == IP_NULL); ip_reference(dest); port->ip_destination = dest; /* now unlock chain */ while (port != base) { ipc_port_t next; /* port is in transit */ assert(ip_active(port)); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination != IP_NULL); next = port->ip_destination; ip_unlock(port); port = next; } /* base is not in transit */ assert(!ip_active(base) || (base->ip_receiver_name != MACH_PORT_NULL) || (base->ip_destination == IP_NULL)); ip_unlock(base); return FALSE; }
boolean_t ipc_port_check_circularity( ipc_port_t port, ipc_port_t dest) { ipc_port_t base; #if IMPORTANCE_INHERITANCE task_t task = TASK_NULL; task_t release_task = TASK_NULL; int assertcnt = 0; #endif /* IMPORTANCE_INHERITANCE */ assert(port != IP_NULL); assert(dest != IP_NULL); if (port == dest) return TRUE; base = dest; /* * First try a quick check that can run in parallel. * No circularity if dest is not in transit. */ ip_lock(port); if (ip_lock_try(dest)) { if (!ip_active(dest) || (dest->ip_receiver_name != MACH_PORT_NULL) || (dest->ip_destination == IP_NULL)) goto not_circular; /* dest is in transit; further checking necessary */ ip_unlock(dest); } ip_unlock(port); ipc_port_multiple_lock(); /* massive serialization */ /* * Search for the end of the chain (a port not in transit), * acquiring locks along the way. */ for (;;) { ip_lock(base); if (!ip_active(base) || (base->ip_receiver_name != MACH_PORT_NULL) || (base->ip_destination == IP_NULL)) break; base = base->ip_destination; } /* all ports in chain from dest to base, inclusive, are locked */ if (port == base) { /* circularity detected! */ ipc_port_multiple_unlock(); /* port (== base) is in limbo */ assert(ip_active(port)); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination == IP_NULL); while (dest != IP_NULL) { ipc_port_t next; /* dest is in transit or in limbo */ assert(ip_active(dest)); assert(dest->ip_receiver_name == MACH_PORT_NULL); next = dest->ip_destination; ip_unlock(dest); dest = next; } return TRUE; } /* * The guarantee: lock port while the entire chain is locked. * Once port is locked, we can take a reference to dest, * add port to the chain, and unlock everything. */ ip_lock(port); ipc_port_multiple_unlock(); not_circular: /* port is in limbo */ assert(ip_active(port)); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination == IP_NULL); ip_reference(dest); port->ip_destination = dest; #if IMPORTANCE_INHERITANCE /* must have been in limbo or still bound to a task */ assert(port->ip_tempowner != 0); if (port->ip_taskptr != 0) { /* * We delayed dropping assertions from a specific task. * Cache that info now (we'll drop assertions and the * task reference below). */ release_task = port->ip_imp_task; port->ip_imp_task = TASK_NULL; port->ip_taskptr = 0; } assertcnt = port->ip_impcount; /* take the port out of limbo w.r.t. assertions */ port->ip_tempowner = 0; #endif /* IMPORTANCE_INHERITANCE */ /* now unlock chain */ ip_unlock(port); for (;;) { #if IMPORTANCE_INHERITANCE /* every port along chain track assertions behind it */ dest->ip_impcount += assertcnt; #endif /* IMPORTANCE_INHERITANCE */ if (dest == base) break; /* port is in transit */ assert(ip_active(dest)); assert(dest->ip_receiver_name == MACH_PORT_NULL); assert(dest->ip_destination != IP_NULL); #if IMPORTANCE_INHERITANCE assert(dest->ip_tempowner == 0); #endif /* IMPORTANCE_INHERITANCE */ port = dest->ip_destination; ip_unlock(dest); dest = port; } /* base is not in transit */ assert(!ip_active(base) || (base->ip_receiver_name != MACH_PORT_NULL) || (base->ip_destination == IP_NULL)); #if IMPORTANCE_INHERITANCE /* * Find the task to boost (if any). * We will boost "through" ports that don't know * about inheritance to deliver receive rights that * do. */ if (ip_active(base) && (assertcnt > 0)) { if (base->ip_tempowner != 0) { if (base->ip_taskptr != 0) /* specified tempowner task */ task = base->ip_imp_task; /* otherwise don't boost current task */ } else if (base->ip_receiver_name != MACH_PORT_NULL) { ipc_space_t space = base->ip_receiver; /* only spaces with boost-accepting tasks */ if (space->is_task != TASK_NULL && space->is_task->imp_receiver != 0) task = space->is_task; } /* take reference before unlocking base */ if (task != TASK_NULL) { assert(task->imp_receiver != 0); task_reference(task); } } #endif /* IMPORTANCE_INHERITANCE */ ip_unlock(base); #if IMPORTANCE_INHERITANCE /* * Transfer assertions now that the ports are unlocked. * Avoid extra overhead if transferring to/from the same task. */ boolean_t transfer_assertions = (task != release_task) ? TRUE : FALSE; if (task != TASK_NULL) { if (transfer_assertions) task_importance_hold_internal_assertion(task, assertcnt); task_deallocate(task); task = TASK_NULL; } if (release_task != TASK_NULL) { if (transfer_assertions) task_importance_drop_internal_assertion(release_task, assertcnt); task_deallocate(release_task); release_task = TASK_NULL; } #endif /* IMPORTANCE_INHERITANCE */ return FALSE; }
void ipc_port_destroy( ipc_port_t port) { ipc_port_t pdrequest, nsrequest; ipc_mqueue_t mqueue; ipc_kmsg_queue_t kmqueue; ipc_kmsg_t kmsg; ipc_thread_t sender; ipc_port_request_t dnrequests; thread_pool_t thread_pool; assert(ip_active(port)); /* port->ip_receiver_name is garbage */ /* port->ip_receiver/port->ip_destination is garbage */ assert(io_otype((ipc_object_t)port) == IOT_PORT); assert(port->ip_pset == IPS_NULL); assert(port->ip_mscount == 0); assert(port->ip_seqno == 0); /* first check for a backup port */ pdrequest = port->ip_pdrequest; if (pdrequest != IP_NULL) { /* we assume the ref for pdrequest */ port->ip_pdrequest = IP_NULL; /* make port be in limbo */ port->ip_receiver_name = MACH_PORT_NAME_NULL; port->ip_destination = IP_NULL; ip_unlock(port); if (!ipc_port_check_circularity(port, pdrequest)) { /* consumes our refs for port and pdrequest */ ipc_notify_port_destroyed(pdrequest, port); return; } else { /* consume pdrequest and destroy port */ ipc_port_release_sonce(pdrequest); } ip_lock(port); assert(ip_active(port)); assert(port->ip_pset == IPS_NULL); assert(port->ip_mscount == 0); assert(port->ip_seqno == 0); assert(port->ip_pdrequest == IP_NULL); assert(port->ip_receiver_name == MACH_PORT_NAME_NULL); assert(port->ip_destination == IP_NULL); /* fall through and destroy the port */ } /* * rouse all blocked senders * * This must be done with the port locked, because * ipc_mqueue_send can play with the ip_blocked queue * of a dead port. */ while ((sender = ipc_thread_dequeue(&port->ip_blocked)) != ITH_NULL) { sender->ith_state = MACH_MSG_SUCCESS; thread_go(sender); } /* once port is dead, we don't need to keep it locked */ port->ip_object.io_bits &= ~IO_BITS_ACTIVE; port->ip_timestamp = ipc_port_timestamp(); /* save for later */ dnrequests = port->ip_dnrequests; port->ip_dnrequests = IPR_NULL; ip_unlock(port); /* wakeup any threads waiting on this pool port for an activation */ if ((thread_pool = &port->ip_thread_pool) != THREAD_POOL_NULL) thread_pool_wakeup(thread_pool); /* throw away no-senders request */ nsrequest = port->ip_nsrequest; if (nsrequest != IP_NULL) ipc_notify_send_once(nsrequest); /* consumes ref */ /* destroy any queued messages */ mqueue = &port->ip_messages; kmqueue = &mqueue->imq_messages; while ((kmsg = ipc_kmsg_dequeue(kmqueue)) != IKM_NULL) { assert(kmsg->ikm_header->msgh_remote_port == (mach_port_t) port); port->ip_msgcount--; ipc_port_release(port); kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; ipc_kmsg_destroy(kmsg); } /* generate dead-name notifications */ if (dnrequests != IPR_NULL) { ipc_port_dnnotify(port, dnrequests); } if (ip_kotype(port) != IKOT_NONE) ipc_kobject_destroy(port); /* XXXX Perhaps should verify that ip_thread_pool is empty! */ ipc_port_release(port); /* consume caller's ref */ }
kern_return_t ipc_right_copyin_two( ipc_space_t space, mach_port_t name, ipc_entry_t entry, ipc_object_t *objectp, ipc_port_t *sorightp) { ipc_entry_bits_t bits = entry->ie_bits; mach_port_urefs_t urefs; ipc_port_t port; ipc_port_t dnrequest = IP_NULL; assert(space->is_active); if ((bits & MACH_PORT_TYPE_SEND) == 0) goto invalid_right; urefs = IE_BITS_UREFS(bits); if (urefs < 2) goto invalid_right; port = (ipc_port_t) entry->ie_object; assert(port != IP_NULL); if (ipc_right_check(space, port, name, entry)) { goto invalid_right; } /* port is locked and active */ assert(port->ip_srights > 0); if (urefs == 2) { if (bits & MACH_PORT_TYPE_RECEIVE) { assert(port->ip_receiver_name == name); assert(port->ip_receiver == space); assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_RECEIVE); port->ip_srights++; ip_reference(port); ip_reference(port); } else { assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND); dnrequest = ipc_right_dncancel_macro(space, port, name, entry); ipc_reverse_remove(space, (ipc_object_t) port); if (bits & IE_BITS_MAREQUEST) ipc_marequest_cancel(space, name); port->ip_srights++; ip_reference(port); entry->ie_object = IO_NULL; } entry->ie_bits = bits &~ (IE_BITS_UREFS_MASK|MACH_PORT_TYPE_SEND); } else { port->ip_srights += 2; ip_reference(port); ip_reference(port); entry->ie_bits = bits-2; /* decrement urefs */ } ip_unlock(port); *objectp = (ipc_object_t) port; *sorightp = dnrequest; return KERN_SUCCESS; invalid_right: return KERN_INVALID_RIGHT; }
void ipc_object_copyout_dest( ipc_space_t space, ipc_object_t object, mach_msg_type_name_t msgt_name, mach_port_name_t *namep) { mach_port_name_t name; assert(IO_VALID(object)); assert(io_active(object)); io_release(object); /* * If the space is the receiver/owner of the object, * then we quietly consume the right and return * the space's name for the object. Otherwise * we destroy the right and return MACH_PORT_NULL. */ switch (msgt_name) { case MACH_MSG_TYPE_PORT_SEND: { ipc_port_t port = (ipc_port_t) object; ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount; if (port->ip_receiver == space) name = port->ip_receiver_name; else name = MACH_PORT_NULL; assert(port->ip_srights > 0); if (--port->ip_srights == 0 && port->ip_nsrequest != IP_NULL) { nsrequest = port->ip_nsrequest; port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; ip_unlock(port); ipc_notify_no_senders(nsrequest, mscount); } else ip_unlock(port); break; } case MACH_MSG_TYPE_PORT_SEND_ONCE: { ipc_port_t port = (ipc_port_t) object; assert(port->ip_sorights > 0); if (port->ip_receiver == space) { /* quietly consume the send-once right */ port->ip_sorights--; name = port->ip_receiver_name; ip_unlock(port); } else { /* * A very bizarre case. The message * was received, but before this copyout * happened the space lost receive rights. * We can't quietly consume the soright * out from underneath some other task, * so generate a send-once notification. */ ip_reference(port); /* restore ref */ ip_unlock(port); ipc_notify_send_once(port); name = MACH_PORT_NULL; } break; } default: panic("ipc_object_copyout_dest: strange rights"); name = MACH_PORT_DEAD; } *namep = name; }
kern_return_t mach_port_space_info( ipc_space_t space, ipc_info_space_t *infop, ipc_info_name_array_t *tablep, mach_msg_type_number_t *tableCntp, __unused ipc_info_tree_name_array_t *treep, __unused mach_msg_type_number_t *treeCntp) { ipc_info_name_t *table_info; vm_offset_t table_addr; vm_size_t table_size, table_size_needed; ipc_entry_t table; ipc_entry_num_t tsize; mach_port_index_t index; kern_return_t kr; vm_map_copy_t copy; if (space == IS_NULL) return KERN_INVALID_TASK; #if !(DEVELOPMENT | DEBUG) const boolean_t dbg_ok = (mac_task_check_expose_task(kernel_task) == 0); #else const boolean_t dbg_ok = TRUE; #endif /* start with in-line memory */ table_size = 0; for (;;) { is_read_lock(space); if (!is_active(space)) { is_read_unlock(space); if (table_size != 0) kmem_free(ipc_kernel_map, table_addr, table_size); return KERN_INVALID_TASK; } table_size_needed = vm_map_round_page((space->is_table_size * sizeof(ipc_info_name_t)), VM_MAP_PAGE_MASK(ipc_kernel_map)); if (table_size_needed == table_size) break; is_read_unlock(space); if (table_size != table_size_needed) { if (table_size != 0) kmem_free(ipc_kernel_map, table_addr, table_size); kr = kmem_alloc(ipc_kernel_map, &table_addr, table_size_needed, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) { return KERN_RESOURCE_SHORTAGE; } table_size = table_size_needed; } } /* space is read-locked and active; we have enough wired memory */ /* get the overall space info */ infop->iis_genno_mask = MACH_PORT_NGEN(MACH_PORT_DEAD); infop->iis_table_size = space->is_table_size; infop->iis_table_next = space->is_table_next->its_size; /* walk the table for this space */ table = space->is_table; tsize = space->is_table_size; table_info = (ipc_info_name_array_t)table_addr; for (index = 0; index < tsize; index++) { ipc_info_name_t *iin = &table_info[index]; ipc_entry_t entry = &table[index]; ipc_entry_bits_t bits; bits = entry->ie_bits; iin->iin_name = MACH_PORT_MAKE(index, IE_BITS_GEN(bits)); iin->iin_collision = 0; iin->iin_type = IE_BITS_TYPE(bits); if ((entry->ie_bits & MACH_PORT_TYPE_PORT_RIGHTS) != MACH_PORT_TYPE_NONE && entry->ie_request != IE_REQ_NONE) { __IGNORE_WCASTALIGN(ipc_port_t port = (ipc_port_t) entry->ie_object); assert(IP_VALID(port)); ip_lock(port); iin->iin_type |= ipc_port_request_type(port, iin->iin_name, entry->ie_request); ip_unlock(port); } iin->iin_urefs = IE_BITS_UREFS(bits); iin->iin_object = (dbg_ok) ? (natural_t)VM_KERNEL_ADDRPERM((uintptr_t)entry->ie_object) : 0; iin->iin_next = entry->ie_next; iin->iin_hash = entry->ie_index; } is_read_unlock(space); /* prepare the table out-of-line data for return */ if (table_size > 0) { vm_size_t used_table_size; used_table_size = infop->iis_table_size * sizeof(ipc_info_name_t); if (table_size > used_table_size) bzero((char *)&table_info[infop->iis_table_size], table_size - used_table_size); kr = vm_map_unwire( ipc_kernel_map, vm_map_trunc_page(table_addr, VM_MAP_PAGE_MASK(ipc_kernel_map)), vm_map_round_page(table_addr + table_size, VM_MAP_PAGE_MASK(ipc_kernel_map)), FALSE); assert(kr == KERN_SUCCESS); kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)table_addr, (vm_map_size_t)used_table_size, TRUE, ©); assert(kr == KERN_SUCCESS); *tablep = (ipc_info_name_t *)copy; *tableCntp = infop->iis_table_size; } else { *tablep = (ipc_info_name_t *)0; *tableCntp = 0; } /* splay tree is obsolete, no work to do... */ *treep = (ipc_info_tree_name_t *)0; *treeCntp = 0; return KERN_SUCCESS; }
boolean_t ipc_port_importance_delta( ipc_port_t port, mach_port_delta_t delta) { ipc_port_t next, base; task_t task = TASK_NULL; boolean_t dropped = FALSE; if (delta == 0) return FALSE; base = port; /* if port is in transit, have to search for end of chain */ if (ip_active(port) && port->ip_destination != IP_NULL && port->ip_receiver_name == MACH_PORT_NULL) { dropped = TRUE; ip_unlock(port); ipc_port_multiple_lock(); /* massive serialization */ ip_lock(base); while(ip_active(base) && base->ip_destination != IP_NULL && base->ip_receiver_name == MACH_PORT_NULL) { base = base->ip_destination; ip_lock(base); } ipc_port_multiple_unlock(); } /* unlock down to the base, adding a boost at each level */ for (;;) { port->ip_impcount += delta; if (port == base) break; /* port is in transit */ assert(port->ip_tempowner == 0); next = port->ip_destination; ip_unlock(port); port = next; } /* find the task (if any) to boost according to the base */ if (ip_active(base)) { if (base->ip_tempowner != 0) { if (base->ip_taskptr != 0) task = base->ip_imp_task; /* otherwise don't boost */ } else if (base->ip_receiver_name != MACH_PORT_NULL) { ipc_space_t space = base->ip_receiver; /* only spaces with boost-accepting tasks */ if (space->is_task != TASK_NULL && space->is_task->imp_receiver != 0) task = space->is_task; } } /* * Only the base is locked. If we have to hold or drop task * importance assertions, we'll have to drop that lock as well. */ if (task != TASK_NULL) { /* take a reference before unlocking base */ assert(task->imp_receiver != 0); task_reference(task); ip_unlock(base); dropped = TRUE; if (delta > 0) task_importance_hold_internal_assertion(task, delta); else task_importance_drop_internal_assertion(task, -delta); task_deallocate(task); } else if (dropped == TRUE) { ip_unlock(base); } return dropped; }
/* * thread_switch: * * Context switch. User may supply thread hint. */ kern_return_t thread_switch( struct thread_switch_args *args) { thread_t thread = THREAD_NULL; thread_t self = current_thread(); mach_port_name_t thread_name = args->thread_name; int option = args->option; mach_msg_timeout_t option_time = args->option_time; uint32_t scale_factor = NSEC_PER_MSEC; boolean_t reenable_workq_callback = FALSE; boolean_t depress_option = FALSE; boolean_t wait_option = FALSE; /* * Validate and process option. */ switch (option) { case SWITCH_OPTION_NONE: workqueue_thread_yielded(); break; case SWITCH_OPTION_WAIT: wait_option = TRUE; workqueue_thread_yielded(); break; case SWITCH_OPTION_DEPRESS: depress_option = TRUE; workqueue_thread_yielded(); break; case SWITCH_OPTION_DISPATCH_CONTENTION: scale_factor = NSEC_PER_USEC; wait_option = TRUE; if (thread_switch_disable_workqueue_sched_callback()) reenable_workq_callback = TRUE; break; case SWITCH_OPTION_OSLOCK_DEPRESS: depress_option = TRUE; if (thread_switch_disable_workqueue_sched_callback()) reenable_workq_callback = TRUE; break; case SWITCH_OPTION_OSLOCK_WAIT: wait_option = TRUE; if (thread_switch_disable_workqueue_sched_callback()) reenable_workq_callback = TRUE; break; default: return (KERN_INVALID_ARGUMENT); } /* * Translate the port name if supplied. */ if (thread_name != MACH_PORT_NULL) { ipc_port_t port; if (ipc_port_translate_send(self->task->itk_space, thread_name, &port) == KERN_SUCCESS) { ip_reference(port); ip_unlock(port); thread = convert_port_to_thread(port); ip_release(port); if (thread == self) { thread_deallocate(thread); thread = THREAD_NULL; } } } if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) { if (thread != THREAD_NULL) { if (thread->task != self->task) { /* * OSLock boosting only applies to other threads * in your same task (even if you have a port for * a thread in another task) */ thread_deallocate(thread); thread = THREAD_NULL; } else { /* * Attempt to kick the lock owner up to our same IO throttling tier. * If the thread is currently blocked in throttle_lowpri_io(), * it will immediately break out. * * TODO: SFI break out? */ int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO); set_thread_iotier_override(thread, new_policy); } } } /* * Try to handoff if supplied. */ if (thread != THREAD_NULL) { spl_t s = splsched(); /* This may return a different thread if the target is pushing on something */ thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread); KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_THREAD_SWITCH)|DBG_FUNC_NONE, thread_tid(thread), thread->state, pulled_thread ? TRUE : FALSE, 0, 0); if (pulled_thread != THREAD_NULL) { /* We can't be dropping the last ref here */ thread_deallocate_safe(thread); if (wait_option) assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor); else if (depress_option) thread_depress_ms(option_time); self->saved.swtch.option = option; self->saved.swtch.reenable_workq_callback = reenable_workq_callback; thread_run(self, (thread_continue_t)thread_switch_continue, NULL, pulled_thread); /* NOTREACHED */ panic("returned from thread_run!"); } splx(s); thread_deallocate(thread); } if (wait_option) assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor); else if (depress_option) thread_depress_ms(option_time); self->saved.swtch.option = option; self->saved.swtch.reenable_workq_callback = reenable_workq_callback; thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD); if (depress_option) thread_depress_abort_internal(self); if (reenable_workq_callback) thread_switch_enable_workqueue_sched_callback(); return (KERN_SUCCESS); }
/* * Routine: ipc_port_spnotify * Purpose: * Generate send-possible port notifications. * Conditions: * Nothing locked, reference held on port. */ void ipc_port_spnotify( ipc_port_t port) { ipc_port_request_index_t index = 0; ipc_table_elems_t size = 0; #if IMPORTANCE_INHERITANCE boolean_t dropassert = FALSE; #endif /* IMPORTANCE_INHERITANCE */ /* * If the port has no send-possible request * armed, don't bother to lock the port. */ if (port->ip_sprequests == 0) return; ip_lock(port); #if IMPORTANCE_INHERITANCE if (port->ip_spimportant != 0) { port->ip_spimportant = 0; port->ip_impcount--; dropassert = TRUE; } #endif /* IMPORTANCE_INHERITANCE */ if (port->ip_sprequests == 0) { ip_unlock(port); goto out; } port->ip_sprequests = 0; revalidate: if (ip_active(port)) { ipc_port_request_t requests; /* table may change each time port unlocked (reload) */ requests = port->ip_requests; assert(requests != IPR_NULL); /* * no need to go beyond table size when first * we entered - those are future notifications. */ if (size == 0) size = requests->ipr_size->its_size; /* no need to backtrack either */ while (++index < size) { ipc_port_request_t ipr = &requests[index]; mach_port_name_t name = ipr->ipr_name; ipc_port_t soright = IPR_SOR_PORT(ipr->ipr_soright); boolean_t armed = IPR_SOR_SPARMED(ipr->ipr_soright); if (MACH_PORT_VALID(name) && armed && IP_VALID(soright)) { /* claim send-once right - slot still inuse */ ipr->ipr_soright = IP_NULL; ip_unlock(port); ipc_notify_send_possible(soright, name); ip_lock(port); goto revalidate; } } } ip_unlock(port); out: #if IMPORTANCE_INHERITANCE if ((dropassert == TRUE) && (current_task()->imp_receiver != 0)) { /* drop internal assertion and no task lock held */ task_importance_drop_internal_assertion(current_task(), 1); } #endif /* IMPORTANCE_INHERITANCE */ return; }
/* * Routine: exception_deliver * Purpose: * Make an upcall to the exception server provided. * Conditions: * Nothing locked and no resources held. * Called from an exception context, so * thread_exception_return and thread_kdb_return * are possible. * Returns: * KERN_SUCCESS if the exception was handled */ kern_return_t exception_deliver( thread_t thread, exception_type_t exception, mach_exception_data_t code, mach_msg_type_number_t codeCnt, struct exception_action *excp, lck_mtx_t *mutex) { ipc_port_t exc_port; exception_data_type_t small_code[EXCEPTION_CODE_MAX]; int code64; int behavior; int flavor; kern_return_t kr; /* * Save work if we are terminating. * Just go back to our AST handler. */ if (!thread->active) return KERN_SUCCESS; /* * Snapshot the exception action data under lock for consistency. * Hold a reference to the port over the exception_raise_* calls * so it can't be destroyed. This seems like overkill, but keeps * the port from disappearing between now and when * ipc_object_copyin_from_kernel is finally called. */ lck_mtx_lock(mutex); exc_port = excp->port; if (!IP_VALID(exc_port)) { lck_mtx_unlock(mutex); return KERN_FAILURE; } ip_lock(exc_port); if (!ip_active(exc_port)) { ip_unlock(exc_port); lck_mtx_unlock(mutex); return KERN_FAILURE; } ip_reference(exc_port); exc_port->ip_srights++; ip_unlock(exc_port); flavor = excp->flavor; behavior = excp->behavior; lck_mtx_unlock(mutex); code64 = (behavior & MACH_EXCEPTION_CODES); behavior &= ~MACH_EXCEPTION_CODES; if (!code64) { small_code[0] = CAST_DOWN_EXPLICIT(exception_data_type_t, code[0]); small_code[1] = CAST_DOWN_EXPLICIT(exception_data_type_t, code[1]); } switch (behavior) { case EXCEPTION_STATE: { mach_msg_type_number_t state_cnt; thread_state_data_t state; c_thr_exc_raise_state++; state_cnt = _MachineStateCount[flavor]; kr = thread_getstatus(thread, flavor, (thread_state_t)state, &state_cnt); if (kr == KERN_SUCCESS) { if (code64) { kr = mach_exception_raise_state(exc_port, exception, code, codeCnt, &flavor, state, state_cnt, state, &state_cnt); } else { kr = exception_raise_state(exc_port, exception, small_code, codeCnt, &flavor, state, state_cnt, state, &state_cnt); } if (kr == MACH_MSG_SUCCESS) kr = thread_setstatus(thread, flavor, (thread_state_t)state, state_cnt); } return kr; } case EXCEPTION_DEFAULT: c_thr_exc_raise++; if (code64) { kr = mach_exception_raise(exc_port, retrieve_thread_self_fast(thread), retrieve_task_self_fast(thread->task), exception, code, codeCnt); } else { kr = exception_raise(exc_port, retrieve_thread_self_fast(thread), retrieve_task_self_fast(thread->task), exception, small_code, codeCnt); } return kr; case EXCEPTION_STATE_IDENTITY: { mach_msg_type_number_t state_cnt; thread_state_data_t state; c_thr_exc_raise_state_id++; state_cnt = _MachineStateCount[flavor]; kr = thread_getstatus(thread, flavor, (thread_state_t)state, &state_cnt); if (kr == KERN_SUCCESS) { if (code64) { kr = mach_exception_raise_state_identity( exc_port, retrieve_thread_self_fast(thread), retrieve_task_self_fast(thread->task), exception, code, codeCnt, &flavor, state, state_cnt, state, &state_cnt); } else { kr = exception_raise_state_identity(exc_port, retrieve_thread_self_fast(thread), retrieve_task_self_fast(thread->task), exception, small_code, codeCnt, &flavor, state, state_cnt, state, &state_cnt); } if (kr == MACH_MSG_SUCCESS) kr = thread_setstatus(thread, flavor, (thread_state_t)state, state_cnt); } return kr; } default: panic ("bad exception behavior!"); return KERN_FAILURE; }/* switch */ }
mach_msg_return_t mach_msg_trap( mach_msg_header_t *msg, mach_msg_option_t option, mach_msg_size_t send_size, mach_msg_size_t rcv_size, mach_port_t rcv_name, mach_msg_timeout_t time_out, mach_port_t notify) { mach_msg_return_t mr; /* first check for common cases */ if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) { ipc_thread_t self = current_thread(); ipc_space_t space = self->task->itk_space; ipc_kmsg_t kmsg; ipc_port_t dest_port; ipc_object_t rcv_object; ipc_mqueue_t rcv_mqueue; mach_msg_size_t reply_size; /* * This case is divided into ten sections, each * with a label. There are five optimized * sections and six unoptimized sections, which * do the same thing but handle all possible * cases and are slower. * * The five sections for an RPC are * 1) Get request message into a buffer. * (fast_get or slow_get) * 2) Copyin request message and rcv_name. * (fast_copyin or slow_copyin) * 3) Enqueue request and dequeue reply. * (fast_send_receive or * slow_send and slow_receive) * 4) Copyout reply message. * (fast_copyout or slow_copyout) * 5) Put reply message to user's buffer. * (fast_put or slow_put) * * Keep the locking hierarchy firmly in mind. * (First spaces, then ports, then port sets, * then message queues.) Only a non-blocking * attempt can be made to acquire locks out of * order, or acquire two locks on the same level. * Acquiring two locks on the same level will * fail if the objects are really the same, * unless simple locking is disabled. This is OK, * because then the extra unlock does nothing. * * There are two major reasons these RPCs can't use * ipc_thread_switch, and use slow_send/slow_receive: * 1) Kernel RPCs. * 2) Servers fall behind clients, so * client doesn't find a blocked server thread and * server finds waiting messages and can't block. */ /* fast_get: */ /* * optimized ipc_kmsg_get * * No locks, references, or messages held. * We must clear ikm_cache before copyinmsg. */ if ((send_size > IKM_SAVED_MSG_SIZE) || (send_size < sizeof(mach_msg_header_t)) || (send_size & 3) || ((kmsg = ikm_cache()) == IKM_NULL)) goto slow_get; ikm_cache() = IKM_NULL; ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE); if (copyinmsg(msg, &kmsg->ikm_header, send_size)) { ikm_free(kmsg); goto slow_get; } kmsg->ikm_header.msgh_size = send_size; fast_copyin: /* * optimized ipc_kmsg_copyin/ipc_mqueue_copyin * * We have the request message data in kmsg. * Must still do copyin, send, receive, etc. * * If the message isn't simple, we can't combine * ipc_kmsg_copyin_header and ipc_mqueue_copyin, * because copyin of the message body might * affect rcv_name. */ switch (kmsg->ikm_header.msgh_bits) { case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND_ONCE): { ipc_entry_t table; ipc_entry_num_t size; ipc_port_t reply_port; /* sending a request message */ { mach_port_index_t index; mach_port_gen_t gen; { mach_port_t reply_name = kmsg->ikm_header.msgh_local_port; if (reply_name != rcv_name) goto slow_copyin; /* optimized ipc_entry_lookup of reply_name */ index = MACH_PORT_INDEX(reply_name); gen = MACH_PORT_GEN(reply_name); } is_read_lock(space); assert(space->is_active); size = space->is_table_size; table = space->is_table; if (index >= size) goto abort_request_copyin; { ipc_entry_t entry; ipc_entry_bits_t bits; entry = &table[index]; bits = entry->ie_bits; /* check generation number and type bit */ if ((bits & (IE_BITS_GEN_MASK| MACH_PORT_TYPE_RECEIVE)) != (gen | MACH_PORT_TYPE_RECEIVE)) goto abort_request_copyin; reply_port = (ipc_port_t) entry->ie_object; assert(reply_port != IP_NULL); } } /* optimized ipc_entry_lookup of dest_name */ { mach_port_index_t index; mach_port_gen_t gen; { mach_port_t dest_name = kmsg->ikm_header.msgh_remote_port; index = MACH_PORT_INDEX(dest_name); gen = MACH_PORT_GEN(dest_name); } if (index >= size) goto abort_request_copyin; { ipc_entry_t entry; ipc_entry_bits_t bits; entry = &table[index]; bits = entry->ie_bits; /* check generation number and type bit */ if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) != (gen | MACH_PORT_TYPE_SEND)) goto abort_request_copyin; assert(IE_BITS_UREFS(bits) > 0); dest_port = (ipc_port_t) entry->ie_object; assert(dest_port != IP_NULL); } } /* * To do an atomic copyin, need simultaneous * locks on both ports and the space. If * dest_port == reply_port, and simple locking is * enabled, then we will abort. Otherwise it's * OK to unlock twice. */ ip_lock(dest_port); if (!ip_active(dest_port) || !ip_lock_try(reply_port)) { ip_unlock(dest_port); goto abort_request_copyin; } is_read_unlock(space); assert(dest_port->ip_srights > 0); dest_port->ip_srights++; ip_reference(dest_port); assert(ip_active(reply_port)); assert(reply_port->ip_receiver_name == kmsg->ikm_header.msgh_local_port); assert(reply_port->ip_receiver == space); reply_port->ip_sorights++; ip_reference(reply_port); kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, MACH_MSG_TYPE_PORT_SEND_ONCE); kmsg->ikm_header.msgh_remote_port = (mach_port_t) dest_port; kmsg->ikm_header.msgh_local_port = (mach_port_t) reply_port; /* make sure we can queue to the destination */ if (dest_port->ip_receiver == ipc_space_kernel) { /* * The kernel server has a reference to * the reply port, which it hands back * to us in the reply message. We do * not need to keep another reference to * it. */ ip_unlock(reply_port); assert(ip_active(dest_port)); ip_unlock(dest_port); goto kernel_send; } if (dest_port->ip_msgcount >= dest_port->ip_qlimit) goto abort_request_send_receive; /* optimized ipc_mqueue_copyin */ if (reply_port->ip_pset != IPS_NULL) goto abort_request_send_receive; rcv_object = (ipc_object_t) reply_port; io_reference(rcv_object); rcv_mqueue = &reply_port->ip_messages; imq_lock(rcv_mqueue); io_unlock(rcv_object); goto fast_send_receive; abort_request_copyin: is_read_unlock(space); goto slow_copyin; abort_request_send_receive: ip_unlock(dest_port); ip_unlock(reply_port); goto slow_send; } case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): { ipc_entry_num_t size; ipc_entry_t table; /* sending a reply message */ { mach_port_t reply_name = kmsg->ikm_header.msgh_local_port; if (reply_name != MACH_PORT_NULL) goto slow_copyin; } is_write_lock(space); assert(space->is_active); /* optimized ipc_entry_lookup */ size = space->is_table_size; table = space->is_table; { ipc_entry_t entry; mach_port_gen_t gen; mach_port_index_t index; { mach_port_t dest_name = kmsg->ikm_header.msgh_remote_port; index = MACH_PORT_INDEX(dest_name); gen = MACH_PORT_GEN(dest_name); } if (index >= size) goto abort_reply_dest_copyin; entry = &table[index]; /* check generation, collision bit, and type bit */ if ((entry->ie_bits & (IE_BITS_GEN_MASK| IE_BITS_COLLISION| MACH_PORT_TYPE_SEND_ONCE)) != (gen | MACH_PORT_TYPE_SEND_ONCE)) goto abort_reply_dest_copyin; /* optimized ipc_right_copyin */ assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_SEND_ONCE); assert(IE_BITS_UREFS(entry->ie_bits) == 1); assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0); if (entry->ie_request != 0) goto abort_reply_dest_copyin; dest_port = (ipc_port_t) entry->ie_object; assert(dest_port != IP_NULL); ip_lock(dest_port); if (!ip_active(dest_port)) { ip_unlock(dest_port); goto abort_reply_dest_copyin; } assert(dest_port->ip_sorights > 0); /* optimized ipc_entry_dealloc */ entry->ie_next = table->ie_next; table->ie_next = index; entry->ie_bits = gen; entry->ie_object = IO_NULL; } kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0); kmsg->ikm_header.msgh_remote_port = (mach_port_t) dest_port; /* make sure we can queue to the destination */ assert(dest_port->ip_receiver != ipc_space_kernel); /* optimized ipc_entry_lookup/ipc_mqueue_copyin */ { ipc_entry_t entry; ipc_entry_bits_t bits; { mach_port_index_t index; mach_port_gen_t gen; index = MACH_PORT_INDEX(rcv_name); gen = MACH_PORT_GEN(rcv_name); if (index >= size) goto abort_reply_rcv_copyin; entry = &table[index]; bits = entry->ie_bits; /* check generation number */ if ((bits & IE_BITS_GEN_MASK) != gen) goto abort_reply_rcv_copyin; } /* check type bits; looking for receive or set */ if (bits & MACH_PORT_TYPE_PORT_SET) { ipc_pset_t rcv_pset; rcv_pset = (ipc_pset_t) entry->ie_object; assert(rcv_pset != IPS_NULL); ips_lock(rcv_pset); assert(ips_active(rcv_pset)); rcv_object = (ipc_object_t) rcv_pset; rcv_mqueue = &rcv_pset->ips_messages; } else if (bits & MACH_PORT_TYPE_RECEIVE) { ipc_port_t rcv_port; rcv_port = (ipc_port_t) entry->ie_object; assert(rcv_port != IP_NULL); if (!ip_lock_try(rcv_port)) goto abort_reply_rcv_copyin; assert(ip_active(rcv_port)); if (rcv_port->ip_pset != IPS_NULL) { ip_unlock(rcv_port); goto abort_reply_rcv_copyin; } rcv_object = (ipc_object_t) rcv_port; rcv_mqueue = &rcv_port->ip_messages; } else goto abort_reply_rcv_copyin; } is_write_unlock(space); io_reference(rcv_object); imq_lock(rcv_mqueue); io_unlock(rcv_object); goto fast_send_receive; abort_reply_dest_copyin: is_write_unlock(space); goto slow_copyin; abort_reply_rcv_copyin: ip_unlock(dest_port); is_write_unlock(space); goto slow_send; } default: goto slow_copyin; } /*NOTREACHED*/ fast_send_receive: /* * optimized ipc_mqueue_send/ipc_mqueue_receive * * Finished get/copyin of kmsg and copyin of rcv_name. * space is unlocked, dest_port is locked, * we can queue kmsg to dest_port, * rcv_mqueue is locked, rcv_object holds a ref, * if rcv_object is a port it isn't in a port set * * Note that if simple locking is turned off, * then we could have dest_mqueue == rcv_mqueue * and not abort when we try to lock dest_mqueue. */ assert(ip_active(dest_port)); assert(dest_port->ip_receiver != ipc_space_kernel); assert((dest_port->ip_msgcount < dest_port->ip_qlimit) || (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE)); assert((kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0); { ipc_mqueue_t dest_mqueue; ipc_thread_t receiver; { ipc_pset_t dest_pset; dest_pset = dest_port->ip_pset; if (dest_pset == IPS_NULL) dest_mqueue = &dest_port->ip_messages; else dest_mqueue = &dest_pset->ips_messages; } if (!imq_lock_try(dest_mqueue)) { abort_send_receive: ip_unlock(dest_port); imq_unlock(rcv_mqueue); ipc_object_release(rcv_object); goto slow_send; } receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads); if ((receiver == ITH_NULL) || (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL)) { imq_unlock(dest_mqueue); goto abort_send_receive; } /* * There is a receiver thread waiting, and * there is no reply message for us to pick up. * We have hope of hand-off, so save state. */ self->ith_msg = msg; self->ith_rcv_size = rcv_size; self->ith_object = rcv_object; self->ith_mqueue = rcv_mqueue; if ((receiver->swap_func == (void (*)()) mach_msg_continue) && thread_handoff(self, mach_msg_continue, receiver)) { assert(current_thread() == receiver); /* * We can use the optimized receive code, * because the receiver is using no options. */ } else if ((receiver->swap_func == (void (*)()) exception_raise_continue) && thread_handoff(self, mach_msg_continue, receiver)) { counter(c_mach_msg_trap_block_exc++); assert(current_thread() == receiver); /* * We are a reply message coming back through * the optimized exception-handling path. * Finish with rcv_mqueue and dest_mqueue, * and then jump to exception code with * dest_port still locked. We don't bother * with a sequence number in this case. */ ipc_thread_enqueue_macro( &rcv_mqueue->imq_threads, self); self->ith_state = MACH_RCV_IN_PROGRESS; self->ith_msize = MACH_MSG_SIZE_MAX; imq_unlock(rcv_mqueue); ipc_thread_rmqueue_first_macro( &dest_mqueue->imq_threads, receiver); imq_unlock(dest_mqueue); exception_raise_continue_fast(dest_port, kmsg); /*NOTREACHED*/ return MACH_MSG_SUCCESS; } else if ((send_size <= receiver->ith_msize) && thread_handoff(self, mach_msg_continue, receiver)) { assert(current_thread() == receiver); if ((receiver->swap_func == (void (*)()) mach_msg_receive_continue) && ((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) { /* * We can still use the optimized code. */ } else { counter(c_mach_msg_trap_block_slow++); /* * We are running as the receiver, * but we can't use the optimized code. * Finish send/receive processing. */ dest_port->ip_msgcount++; ip_unlock(dest_port); ipc_thread_enqueue_macro( &rcv_mqueue->imq_threads, self); self->ith_state = MACH_RCV_IN_PROGRESS; self->ith_msize = MACH_MSG_SIZE_MAX; imq_unlock(rcv_mqueue); ipc_thread_rmqueue_first_macro( &dest_mqueue->imq_threads, receiver); receiver->ith_state = MACH_MSG_SUCCESS; receiver->ith_kmsg = kmsg; receiver->ith_seqno = dest_port->ip_seqno++; imq_unlock(dest_mqueue); /* * Call the receiver's continuation. */ receiver->wait_result = THREAD_AWAKENED; (*receiver->swap_func)(); /*NOTREACHED*/ return MACH_MSG_SUCCESS; } } else { /* * The receiver can't accept the message, * or we can't switch to the receiver. */ imq_unlock(dest_mqueue); goto abort_send_receive; } counter(c_mach_msg_trap_block_fast++); /* * Safe to unlock dest_port now that we are * committed to this path, because we hold * dest_mqueue locked. We never bother changing * dest_port->ip_msgcount. */ ip_unlock(dest_port); /* * We need to finish preparing self for its * time asleep in rcv_mqueue. */ ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self); self->ith_state = MACH_RCV_IN_PROGRESS; self->ith_msize = MACH_MSG_SIZE_MAX; imq_unlock(rcv_mqueue); /* * Finish extracting receiver from dest_mqueue. */ ipc_thread_rmqueue_first_macro( &dest_mqueue->imq_threads, receiver); kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++; imq_unlock(dest_mqueue); /* * We don't have to do any post-dequeue processing of * the message. We never incremented ip_msgcount, we * know it has no msg-accepted request, and blocked * senders aren't a worry because we found the port * with a receiver waiting. */ self = receiver; space = self->task->itk_space; msg = self->ith_msg; rcv_size = self->ith_rcv_size; rcv_object = self->ith_object; /* inline ipc_object_release */ io_lock(rcv_object); io_release(rcv_object); io_check_unlock(rcv_object); } fast_copyout: /* * Nothing locked and no references held, except * we have kmsg with msgh_seqno filled in. Must * still check against rcv_size and do * ipc_kmsg_copyout/ipc_kmsg_put. */ assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port == dest_port); reply_size = kmsg->ikm_header.msgh_size; if (rcv_size < reply_size) goto slow_copyout; /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */ switch (kmsg->ikm_header.msgh_bits) { case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, MACH_MSG_TYPE_PORT_SEND_ONCE): { ipc_port_t reply_port = (ipc_port_t) kmsg->ikm_header.msgh_local_port; mach_port_t dest_name, reply_name; /* receiving a request message */ if (!IP_VALID(reply_port)) goto slow_copyout; is_write_lock(space); assert(space->is_active); /* * To do an atomic copyout, need simultaneous * locks on both ports and the space. If * dest_port == reply_port, and simple locking is * enabled, then we will abort. Otherwise it's * OK to unlock twice. */ ip_lock(dest_port); if (!ip_active(dest_port) || !ip_lock_try(reply_port)) goto abort_request_copyout; if (!ip_active(reply_port)) { ip_unlock(reply_port); goto abort_request_copyout; } assert(reply_port->ip_sorights > 0); ip_unlock(reply_port); { ipc_entry_t table; ipc_entry_t entry; mach_port_index_t index; /* optimized ipc_entry_get */ table = space->is_table; index = table->ie_next; if (index == 0) goto abort_request_copyout; entry = &table[index]; table->ie_next = entry->ie_next; entry->ie_request = 0; { mach_port_gen_t gen; assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0); gen = entry->ie_bits + IE_BITS_GEN_ONE; reply_name = MACH_PORT_MAKE(index, gen); /* optimized ipc_right_copyout */ entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1); } assert(MACH_PORT_VALID(reply_name)); entry->ie_object = (ipc_object_t) reply_port; is_write_unlock(space); } /* optimized ipc_object_copyout_dest */ assert(dest_port->ip_srights > 0); ip_release(dest_port); if (dest_port->ip_receiver == space) dest_name = dest_port->ip_receiver_name; else dest_name = MACH_PORT_NULL; if ((--dest_port->ip_srights == 0) && (dest_port->ip_nsrequest != IP_NULL)) { ipc_port_t nsrequest; mach_port_mscount_t mscount; /* a rather rare case */ nsrequest = dest_port->ip_nsrequest; mscount = dest_port->ip_mscount; dest_port->ip_nsrequest = IP_NULL; ip_unlock(dest_port); ipc_notify_no_senders(nsrequest, mscount); } else ip_unlock(dest_port); if (! ipc_port_flag_protected_payload(dest_port)) { kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS( MACH_MSG_TYPE_PORT_SEND_ONCE, MACH_MSG_TYPE_PORT_SEND); kmsg->ikm_header.msgh_local_port = dest_name; } else { kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS( MACH_MSG_TYPE_PORT_SEND_ONCE, MACH_MSG_TYPE_PROTECTED_PAYLOAD); kmsg->ikm_header.msgh_protected_payload = dest_port->ip_protected_payload; } kmsg->ikm_header.msgh_remote_port = reply_name; goto fast_put; abort_request_copyout: ip_unlock(dest_port); is_write_unlock(space); goto slow_copyout; } case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): { mach_port_t dest_name; /* receiving a reply message */ ip_lock(dest_port); if (!ip_active(dest_port)) goto slow_copyout; /* optimized ipc_object_copyout_dest */ assert(dest_port->ip_sorights > 0); if (dest_port->ip_receiver == space) { ip_release(dest_port); dest_port->ip_sorights--; dest_name = dest_port->ip_receiver_name; ip_unlock(dest_port); } else { ip_unlock(dest_port); ipc_notify_send_once(dest_port); dest_name = MACH_PORT_NULL; } if (! ipc_port_flag_protected_payload(dest_port)) { kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS( 0, MACH_MSG_TYPE_PORT_SEND_ONCE); kmsg->ikm_header.msgh_local_port = dest_name; } else { kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS( 0, MACH_MSG_TYPE_PROTECTED_PAYLOAD); kmsg->ikm_header.msgh_protected_payload = dest_port->ip_protected_payload; } kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL; goto fast_put; } case MACH_MSGH_BITS_COMPLEX| MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): { mach_port_t dest_name; /* receiving a complex reply message */ ip_lock(dest_port); if (!ip_active(dest_port)) goto slow_copyout; /* optimized ipc_object_copyout_dest */ assert(dest_port->ip_sorights > 0); if (dest_port->ip_receiver == space) { ip_release(dest_port); dest_port->ip_sorights--; dest_name = dest_port->ip_receiver_name; ip_unlock(dest_port); } else { ip_unlock(dest_port); ipc_notify_send_once(dest_port); dest_name = MACH_PORT_NULL; } if (! ipc_port_flag_protected_payload(dest_port)) { kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS_COMPLEX | MACH_MSGH_BITS( 0, MACH_MSG_TYPE_PORT_SEND_ONCE); kmsg->ikm_header.msgh_local_port = dest_name; } else { kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS_COMPLEX | MACH_MSGH_BITS( 0, MACH_MSG_TYPE_PROTECTED_PAYLOAD); kmsg->ikm_header.msgh_protected_payload = dest_port->ip_protected_payload; } kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL; mr = ipc_kmsg_copyout_body( (vm_offset_t) (&kmsg->ikm_header + 1), (vm_offset_t) &kmsg->ikm_header + kmsg->ikm_header.msgh_size, space, current_map()); if (mr != MACH_MSG_SUCCESS) { (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); return mr | MACH_RCV_BODY_ERROR; } goto fast_put; } default: goto slow_copyout; } /*NOTREACHED*/ fast_put: /* * We have the reply message data in kmsg, * and the reply message size in reply_size. * Just need to copy it out to the user and free kmsg. * We must check ikm_cache after copyoutmsg. */ ikm_check_initialized(kmsg, kmsg->ikm_size); if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) || copyoutmsg(&kmsg->ikm_header, msg, reply_size) || (ikm_cache() != IKM_NULL)) goto slow_put; ikm_cache() = kmsg; thread_syscall_return(MACH_MSG_SUCCESS); /*NOTREACHED*/ return MACH_MSG_SUCCESS; /* help for the compiler */ /* * The slow path has a few non-register temporary * variables used only for call-by-reference. */ { ipc_kmsg_t temp_kmsg; mach_port_seqno_t temp_seqno; ipc_object_t temp_rcv_object; ipc_mqueue_t temp_rcv_mqueue; slow_get: /* * No locks, references, or messages held. * Still have to get the request, send it, * receive reply, etc. */ mr = ipc_kmsg_get(msg, send_size, &temp_kmsg); if (mr != MACH_MSG_SUCCESS) { thread_syscall_return(mr); /*NOTREACHED*/ } kmsg = temp_kmsg; /* try to get back on optimized path */ goto fast_copyin; slow_copyin: /* * We have the message data in kmsg, but * we still need to copyin, send it, * receive a reply, and do copyout. */ mr = ipc_kmsg_copyin(kmsg, space, current_map(), MACH_PORT_NULL); if (mr != MACH_MSG_SUCCESS) { ikm_free(kmsg); thread_syscall_return(mr); /*NOTREACHED*/ } /* try to get back on optimized path */ if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR) goto slow_send; dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port; assert(IP_VALID(dest_port)); ip_lock(dest_port); if (dest_port->ip_receiver == ipc_space_kernel) { assert(ip_active(dest_port)); ip_unlock(dest_port); goto kernel_send; } if (ip_active(dest_port) && ((dest_port->ip_msgcount < dest_port->ip_qlimit) || (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE))) { /* * Try an optimized ipc_mqueue_copyin. * It will work if this is a request message. */ ipc_port_t reply_port; reply_port = (ipc_port_t) kmsg->ikm_header.msgh_local_port; if (IP_VALID(reply_port)) { if (ip_lock_try(reply_port)) { if (ip_active(reply_port) && reply_port->ip_receiver == space && reply_port->ip_receiver_name == rcv_name && reply_port->ip_pset == IPS_NULL) { /* Grab a reference to the reply port. */ rcv_object = (ipc_object_t) reply_port; io_reference(rcv_object); rcv_mqueue = &reply_port->ip_messages; imq_lock(rcv_mqueue); io_unlock(rcv_object); goto fast_send_receive; } ip_unlock(reply_port); } } } ip_unlock(dest_port); goto slow_send; kernel_send: /* * Special case: send message to kernel services. * The request message has been copied into the * kmsg. Nothing is locked. */ { ipc_port_t reply_port; /* * Perform the kernel function. */ kmsg = ipc_kobject_server(kmsg); if (kmsg == IKM_NULL) { /* * No reply. Take the * slow receive path. */ goto slow_get_rcv_port; } /* * Check that: * the reply port is alive * we hold the receive right * the name has not changed. * the port is not in a set * If any of these are not true, * we cannot directly receive the reply * message. */ reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port; ip_lock(reply_port); if ((!ip_active(reply_port)) || (reply_port->ip_receiver != space) || (reply_port->ip_receiver_name != rcv_name) || (reply_port->ip_pset != IPS_NULL)) { ip_unlock(reply_port); ipc_mqueue_send_always(kmsg); goto slow_get_rcv_port; } rcv_mqueue = &reply_port->ip_messages; imq_lock(rcv_mqueue); /* keep port locked, and don`t change ref count yet */ /* * If there are messages on the port * or other threads waiting for a message, * we cannot directly receive the reply. */ if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads) != ITH_NULL) || (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL)) { imq_unlock(rcv_mqueue); ip_unlock(reply_port); ipc_mqueue_send_always(kmsg); goto slow_get_rcv_port; } /* * We can directly receive this reply. * Since the kernel reply never blocks, * it holds no message_accepted request. * Since there were no messages queued * on the reply port, there should be * no threads blocked waiting to send. */ assert(kmsg->ikm_marequest == IMAR_NULL); assert(ipc_thread_queue_first(&reply_port->ip_blocked) == ITH_NULL); dest_port = reply_port; kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++; imq_unlock(rcv_mqueue); /* * inline ipc_object_release. * Port is still locked. * Reference count was not incremented. */ ip_check_unlock(reply_port); /* copy out the kernel reply */ goto fast_copyout; } slow_send: /* * Nothing is locked. We have acquired kmsg, but * we still need to send it and receive a reply. */ mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE, MACH_MSG_TIMEOUT_NONE); if (mr != MACH_MSG_SUCCESS) { mr |= ipc_kmsg_copyout_pseudo(kmsg, space, current_map()); assert(kmsg->ikm_marequest == IMAR_NULL); (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); thread_syscall_return(mr); /*NOTREACHED*/ } slow_get_rcv_port: /* * We have sent the message. Copy in the receive port. */ mr = ipc_mqueue_copyin(space, rcv_name, &temp_rcv_mqueue, &temp_rcv_object); if (mr != MACH_MSG_SUCCESS) { thread_syscall_return(mr); /*NOTREACHED*/ } rcv_mqueue = temp_rcv_mqueue; rcv_object = temp_rcv_object; /* hold ref for rcv_object; rcv_mqueue is locked */ /* slow_receive: */ /* * Now we have sent the request and copied in rcv_name, * so rcv_mqueue is locked and hold ref for rcv_object. * Just receive a reply and try to get back to fast path. * * ipc_mqueue_receive may not return, because if we block * then our kernel stack may be discarded. So we save * state here for mach_msg_continue to pick up. */ self->ith_msg = msg; self->ith_rcv_size = rcv_size; self->ith_object = rcv_object; self->ith_mqueue = rcv_mqueue; mr = ipc_mqueue_receive(rcv_mqueue, MACH_MSG_OPTION_NONE, MACH_MSG_SIZE_MAX, MACH_MSG_TIMEOUT_NONE, FALSE, mach_msg_continue, &temp_kmsg, &temp_seqno); /* rcv_mqueue is unlocked */ ipc_object_release(rcv_object); if (mr != MACH_MSG_SUCCESS) { thread_syscall_return(mr); /*NOTREACHED*/ } (kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno; dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port; goto fast_copyout; slow_copyout: /* * Nothing locked and no references held, except * we have kmsg with msgh_seqno filled in. Must * still check against rcv_size and do * ipc_kmsg_copyout/ipc_kmsg_put. */ reply_size = kmsg->ikm_header.msgh_size; if (rcv_size < reply_size) { ipc_kmsg_copyout_dest(kmsg, space); (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); thread_syscall_return(MACH_RCV_TOO_LARGE); /*NOTREACHED*/ } mr = ipc_kmsg_copyout(kmsg, space, current_map(), MACH_PORT_NULL); if (mr != MACH_MSG_SUCCESS) { if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); } else { ipc_kmsg_copyout_dest(kmsg, space); (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); } thread_syscall_return(mr); /*NOTREACHED*/ } /* try to get back on optimized path */ goto fast_put; slow_put: mr = ipc_kmsg_put(msg, kmsg, reply_size); thread_syscall_return(mr); /*NOTREACHED*/ } } else if (option == MACH_SEND_MSG) { ipc_space_t space = current_space(); vm_map_t map = current_map(); ipc_kmsg_t kmsg; mr = ipc_kmsg_get(msg, send_size, &kmsg); if (mr != MACH_MSG_SUCCESS) return mr; mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL); if (mr != MACH_MSG_SUCCESS) { ikm_free(kmsg); return mr; } mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE, MACH_MSG_TIMEOUT_NONE); if (mr != MACH_MSG_SUCCESS) { mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map); assert(kmsg->ikm_marequest == IMAR_NULL); (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); } return mr; } else if (option == MACH_RCV_MSG) { ipc_thread_t self = current_thread(); ipc_space_t space = current_space(); vm_map_t map = current_map(); ipc_object_t object; ipc_mqueue_t mqueue; ipc_kmsg_t kmsg; mach_port_seqno_t seqno; mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object); if (mr != MACH_MSG_SUCCESS) return mr; /* hold ref for object; mqueue is locked */ /* * ipc_mqueue_receive may not return, because if we block * then our kernel stack may be discarded. So we save * state here for mach_msg_continue to pick up. */ self->ith_msg = msg; self->ith_rcv_size = rcv_size; self->ith_object = object; self->ith_mqueue = mqueue; mr = ipc_mqueue_receive(mqueue, MACH_MSG_OPTION_NONE, MACH_MSG_SIZE_MAX, MACH_MSG_TIMEOUT_NONE, FALSE, mach_msg_continue, &kmsg, &seqno); /* mqueue is unlocked */ ipc_object_release(object); if (mr != MACH_MSG_SUCCESS) return mr; kmsg->ikm_header.msgh_seqno = seqno; if (rcv_size < kmsg->ikm_header.msgh_size) { ipc_kmsg_copyout_dest(kmsg, space); (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); return MACH_RCV_TOO_LARGE; } mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL); if (mr != MACH_MSG_SUCCESS) { if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); } else { ipc_kmsg_copyout_dest(kmsg, space); (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); } return mr; } return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); } else if (option == MACH_MSG_OPTION_NONE) { /* * We can measure the "null mach_msg_trap" * (syscall entry and thread_syscall_return exit) * with this path. */ thread_syscall_return(MACH_MSG_SUCCESS); /*NOTREACHED*/ } if (option & MACH_SEND_MSG) { mr = mach_msg_send(msg, option, send_size, time_out, notify); if (mr != MACH_MSG_SUCCESS) return mr; } if (option & MACH_RCV_MSG) { mr = mach_msg_receive(msg, option, rcv_size, rcv_name, time_out, notify); if (mr != MACH_MSG_SUCCESS) return mr; } return MACH_MSG_SUCCESS; }
wait_result_t ipc_mqueue_receive_on_thread( ipc_mqueue_t mqueue, mach_msg_option_t option, mach_msg_size_t max_size, mach_msg_timeout_t rcv_timeout, int interruptible, thread_t thread) { ipc_kmsg_queue_t kmsgs; wait_result_t wresult; uint64_t deadline; spl_t s; #if CONFIG_MACF_MACH ipc_labelh_t lh; task_t task; int rc; #endif s = splsched(); imq_lock(mqueue); if (imq_is_set(mqueue)) { queue_t q; q = &mqueue->imq_preposts; /* * If we are waiting on a portset mqueue, we need to see if * any of the member ports have work for us. Ports that * have (or recently had) messages will be linked in the * prepost queue for the portset. By holding the portset's * mqueue lock during the search, we tie up any attempts by * mqueue_deliver or portset membership changes that may * cross our path. */ search_set: while(!queue_empty(q)) { wait_queue_link_t wql; ipc_mqueue_t port_mq; queue_remove_first(q, wql, wait_queue_link_t, wql_preposts); assert(!wql_is_preposted(wql)); /* * This is a lock order violation, so we have to do it * "softly," putting the link back on the prepost list * if it fails (at the tail is fine since the order of * handling messages from different sources in a set is * not guaranteed and we'd like to skip to the next source * if one is available). */ port_mq = (ipc_mqueue_t)wql->wql_queue; if (!imq_lock_try(port_mq)) { queue_enter(q, wql, wait_queue_link_t, wql_preposts); imq_unlock(mqueue); splx(s); mutex_pause(0); s = splsched(); imq_lock(mqueue); goto search_set; /* start again at beginning - SMP */ } /* * If there are no messages on this queue, just skip it * (we already removed the link from the set's prepost queue). */ kmsgs = &port_mq->imq_messages; if (ipc_kmsg_queue_first(kmsgs) == IKM_NULL) { imq_unlock(port_mq); continue; } /* * There are messages, so reinsert the link back * at the tail of the preposted queue (for fairness) * while we still have the portset mqueue locked. */ queue_enter(q, wql, wait_queue_link_t, wql_preposts); imq_unlock(mqueue); /* * Continue on to handling the message with just * the port mqueue locked. */ ipc_mqueue_select_on_thread(port_mq, option, max_size, thread); imq_unlock(port_mq); #if CONFIG_MACF_MACH if (thread->task != TASK_NULL && thread->ith_kmsg != NULL && thread->ith_kmsg->ikm_sender != NULL) { lh = thread->ith_kmsg->ikm_sender->label; tasklabel_lock(thread->task); ip_lock(lh->lh_port); rc = mac_port_check_receive(&thread->task->maclabel, &lh->lh_label); ip_unlock(lh->lh_port); tasklabel_unlock(thread->task); if (rc) thread->ith_state = MACH_RCV_INVALID_DATA; } #endif splx(s); return THREAD_NOT_WAITING; } } else { /* * Receive on a single port. Just try to get the messages. */ kmsgs = &mqueue->imq_messages; if (ipc_kmsg_queue_first(kmsgs) != IKM_NULL) { ipc_mqueue_select_on_thread(mqueue, option, max_size, thread); imq_unlock(mqueue); #if CONFIG_MACF_MACH if (thread->task != TASK_NULL && thread->ith_kmsg != NULL && thread->ith_kmsg->ikm_sender != NULL) { lh = thread->ith_kmsg->ikm_sender->label; tasklabel_lock(thread->task); ip_lock(lh->lh_port); rc = mac_port_check_receive(&thread->task->maclabel, &lh->lh_label); ip_unlock(lh->lh_port); tasklabel_unlock(thread->task); if (rc) thread->ith_state = MACH_RCV_INVALID_DATA; } #endif splx(s); return THREAD_NOT_WAITING; } } /* * Looks like we'll have to block. The mqueue we will * block on (whether the set's or the local port's) is * still locked. */ if (option & MACH_RCV_TIMEOUT) { if (rcv_timeout == 0) { imq_unlock(mqueue); splx(s); thread->ith_state = MACH_RCV_TIMED_OUT; return THREAD_NOT_WAITING; } } thread_lock(thread); thread->ith_state = MACH_RCV_IN_PROGRESS; thread->ith_option = option; thread->ith_msize = max_size; if (option & MACH_RCV_TIMEOUT) clock_interval_to_deadline(rcv_timeout, 1000*NSEC_PER_USEC, &deadline); else deadline = 0; wresult = wait_queue_assert_wait64_locked(&mqueue->imq_wait_queue, IPC_MQUEUE_RECEIVE, interruptible, deadline, thread); /* preposts should be detected above, not here */ if (wresult == THREAD_AWAKENED) panic("ipc_mqueue_receive_on_thread: sleep walking"); thread_unlock(thread); imq_unlock(mqueue); splx(s); return wresult; }