/*
 * Register a new module with the framework.
 * This registers interest in changes to the set of netstacks.
 * The createfn and destroyfn are required, but the shutdownfn can be
 * NULL.
 * Note that due to the current zsd implementation, when the create
 * function is called the zone isn't fully present, thus functions
 * like zone_find_by_* will fail, hence the create function can not
 * use many zones kernel functions including zcmn_err().
 */
void
netstack_register(int moduleid,
    void *(*module_create)(netstackid_t, netstack_t *),
    void (*module_shutdown)(netstackid_t, void *),
    void (*module_destroy)(netstackid_t, void *))
{
	netstack_t *ns;

	ASSERT(netstack_initialized);
	ASSERT(moduleid >= 0 && moduleid < NS_MAX);
	ASSERT(module_create != NULL);

	/*
	 * Make instances created after this point in time run the create
	 * callback.
	 */
	mutex_enter(&netstack_g_lock);
	ASSERT(ns_reg[moduleid].nr_create == NULL);
	ASSERT(ns_reg[moduleid].nr_flags == 0);
	ns_reg[moduleid].nr_create = module_create;
	ns_reg[moduleid].nr_shutdown = module_shutdown;
	ns_reg[moduleid].nr_destroy = module_destroy;
	ns_reg[moduleid].nr_flags = NRF_REGISTERED;

	/*
	 * Determine the set of stacks that exist before we drop the lock.
	 * Set NSS_CREATE_NEEDED for each of those.
	 * netstacks which have been deleted will have NSS_CREATE_COMPLETED
	 * set, but check NSF_CLOSING to be sure.
	 */
	for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
		nm_state_t *nms = &ns->netstack_m_state[moduleid];

		mutex_enter(&ns->netstack_lock);
		if (!(ns->netstack_flags & NSF_CLOSING) &&
		    (nms->nms_flags & NSS_CREATE_ALL) == 0) {
			nms->nms_flags |= NSS_CREATE_NEEDED;
			DTRACE_PROBE2(netstack__create__needed,
			    netstack_t *, ns, int, moduleid);
		}
		mutex_exit(&ns->netstack_lock);
	}
	mutex_exit(&netstack_g_lock);

	/*
	 * At this point in time a new instance can be created or an instance
	 * can be destroyed, or some other module can register or unregister.
	 * Make sure we either run all the create functions for this moduleid
	 * or we wait for any other creators for this moduleid.
	 */
	apply_all_netstacks(moduleid, netstack_apply_create);
}
void
netstack_unregister(int moduleid)
{
	netstack_t *ns;

	ASSERT(moduleid >= 0 && moduleid < NS_MAX);

	ASSERT(ns_reg[moduleid].nr_create != NULL);
	ASSERT(ns_reg[moduleid].nr_flags & NRF_REGISTERED);

	mutex_enter(&netstack_g_lock);
	/*
	 * Determine the set of stacks that exist before we drop the lock.
	 * Set NSS_SHUTDOWN_NEEDED and NSS_DESTROY_NEEDED for each of those.
	 * That ensures that when we return all the callbacks for existing
	 * instances have completed. And since we set NRF_DYING no new
	 * instances can use this module.
	 */
	for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
		boolean_t created = B_FALSE;
		nm_state_t *nms = &ns->netstack_m_state[moduleid];

		mutex_enter(&ns->netstack_lock);

		/*
		 * We need to be careful here. We could actually have a netstack
		 * being created as we speak waiting for us to let go of this
		 * lock to proceed. It may have set NSS_CREATE_NEEDED, but not
		 * have gotten to the point of completing it yet. If
		 * NSS_CREATE_NEEDED, we can safely just remove it here and
		 * never create the module. However, if NSS_CREATE_INPROGRESS is
		 * set, we need to still flag this module for shutdown and
		 * deletion, just as though it had reached NSS_CREATE_COMPLETED.
		 *
		 * It is safe to do that because of two different guarantees
		 * that exist in the system. The first is that before we do a
		 * create, shutdown, or destroy, we ensure that nothing else is
		 * in progress in the system for this netstack and wait for it
		 * to complete. Secondly, because the zone is being created, we
		 * know that the following call to apply_all_netstack will block
		 * on the zone finishing its initialization.
		 */
		if (nms->nms_flags & NSS_CREATE_NEEDED)
			nms->nms_flags &= ~NSS_CREATE_NEEDED;

		if (nms->nms_flags & NSS_CREATE_INPROGRESS ||
		    nms->nms_flags & NSS_CREATE_COMPLETED)
			created = B_TRUE;

		if (ns_reg[moduleid].nr_shutdown != NULL && created &&
		    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
		    (nms->nms_flags & NSS_SHUTDOWN_ALL) == 0) {
			nms->nms_flags |= NSS_SHUTDOWN_NEEDED;
			DTRACE_PROBE2(netstack__shutdown__needed,
			    netstack_t *, ns, int, moduleid);
		}
		if ((ns_reg[moduleid].nr_flags & NRF_REGISTERED) &&
		    ns_reg[moduleid].nr_destroy != NULL && created &&
		    (nms->nms_flags & NSS_DESTROY_ALL) == 0) {
			nms->nms_flags |= NSS_DESTROY_NEEDED;
			DTRACE_PROBE2(netstack__destroy__needed,
			    netstack_t *, ns, int, moduleid);
		}
		mutex_exit(&ns->netstack_lock);
	}
	/*
	 * Prevent any new netstack from calling the registered create
	 * function, while keeping the function pointers in place until the
	 * shutdown and destroy callbacks are complete.
	 */
	ns_reg[moduleid].nr_flags |= NRF_DYING;
	mutex_exit(&netstack_g_lock);

	apply_all_netstacks(moduleid, netstack_apply_shutdown);
	apply_all_netstacks(moduleid, netstack_apply_destroy);

	/*
	 * Clear the nms_flags so that we can handle this module
	 * being loaded again.
	 * Also remove the registered functions.
	 */
	mutex_enter(&netstack_g_lock);
	ASSERT(ns_reg[moduleid].nr_flags & NRF_REGISTERED);
	ASSERT(ns_reg[moduleid].nr_flags & NRF_DYING);
	for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
		nm_state_t *nms = &ns->netstack_m_state[moduleid];

		mutex_enter(&ns->netstack_lock);
		if (nms->nms_flags & NSS_DESTROY_COMPLETED) {
			nms->nms_flags = 0;
			DTRACE_PROBE2(netstack__destroy__done,
			    netstack_t *, ns, int, moduleid);
		}
		mutex_exit(&ns->netstack_lock);
	}

	ns_reg[moduleid].nr_create = NULL;
	ns_reg[moduleid].nr_shutdown = NULL;
	ns_reg[moduleid].nr_destroy = NULL;
	ns_reg[moduleid].nr_flags = 0;
	mutex_exit(&netstack_g_lock);
}
void
netstack_unregister(int moduleid)
{
	netstack_t *ns;

	ASSERT(moduleid >= 0 && moduleid < NS_MAX);

	ASSERT(ns_reg[moduleid].nr_create != NULL);
	ASSERT(ns_reg[moduleid].nr_flags & NRF_REGISTERED);

	mutex_enter(&netstack_g_lock);
	/*
	 * Determine the set of stacks that exist before we drop the lock.
	 * Set NSS_SHUTDOWN_NEEDED and NSS_DESTROY_NEEDED for each of those.
	 * That ensures that when we return all the callbacks for existing
	 * instances have completed. And since we set NRF_DYING no new
	 * instances can use this module.
	 */
	for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
		nm_state_t *nms = &ns->netstack_m_state[moduleid];

		mutex_enter(&ns->netstack_lock);
		if (ns_reg[moduleid].nr_shutdown != NULL &&
		    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
		    (nms->nms_flags & NSS_SHUTDOWN_ALL) == 0) {
			nms->nms_flags |= NSS_SHUTDOWN_NEEDED;
			DTRACE_PROBE2(netstack__shutdown__needed,
			    netstack_t *, ns, int, moduleid);
		}
		if ((ns_reg[moduleid].nr_flags & NRF_REGISTERED) &&
		    ns_reg[moduleid].nr_destroy != NULL &&
		    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
		    (nms->nms_flags & NSS_DESTROY_ALL) == 0) {
			nms->nms_flags |= NSS_DESTROY_NEEDED;
			DTRACE_PROBE2(netstack__destroy__needed,
			    netstack_t *, ns, int, moduleid);
		}
		mutex_exit(&ns->netstack_lock);
	}
	/*
	 * Prevent any new netstack from calling the registered create
	 * function, while keeping the function pointers in place until the
	 * shutdown and destroy callbacks are complete.
	 */
	ns_reg[moduleid].nr_flags |= NRF_DYING;
	mutex_exit(&netstack_g_lock);

	apply_all_netstacks(moduleid, netstack_apply_shutdown);
	apply_all_netstacks(moduleid, netstack_apply_destroy);

	/*
	 * Clear the nms_flags so that we can handle this module
	 * being loaded again.
	 * Also remove the registered functions.
	 */
	mutex_enter(&netstack_g_lock);
	ASSERT(ns_reg[moduleid].nr_flags & NRF_REGISTERED);
	ASSERT(ns_reg[moduleid].nr_flags & NRF_DYING);
	for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
		nm_state_t *nms = &ns->netstack_m_state[moduleid];

		mutex_enter(&ns->netstack_lock);
		if (nms->nms_flags & NSS_DESTROY_COMPLETED) {
			nms->nms_flags = 0;
			DTRACE_PROBE2(netstack__destroy__done,
			    netstack_t *, ns, int, moduleid);
		}
		mutex_exit(&ns->netstack_lock);
	}

	ns_reg[moduleid].nr_create = NULL;
	ns_reg[moduleid].nr_shutdown = NULL;
	ns_reg[moduleid].nr_destroy = NULL;
	ns_reg[moduleid].nr_flags = 0;
	mutex_exit(&netstack_g_lock);
}