void
mdeg_fini(void)
{
/*
* Flip the enabled switch off to make sure that
* no events get dispatched while things are being
* torn down.
*/
mdeg.enabled = B_FALSE;
/* destroy the task queue */
taskq_destroy(mdeg.taskq);
/*
* Deallocate the table of registered clients
*/
kmem_free(mdeg.tbl, mdeg.maxclnts * sizeof (mdeg_clnt_t));
rw_destroy(&mdeg.rwlock);
/*
* Free up the cached MDs.
*/
if (mdeg.md_curr)
(void) md_fini_handle(mdeg.md_curr);
if (mdeg.md_prev)
(void) md_fini_handle(mdeg.md_prev);
mutex_destroy(&mdeg.lock);
}
/*
* General purpose initialization function used to extract parameters
* from the MD during the boot process. This is called immediately after
* the in kernel copy of the MD has been initialized so that global
* flags are available to various subsystems as they get initialized.
*/
static void
init_md_params(void)
{
md_t *mdp;
int num_nodes;
mde_cookie_t *listp;
int listsz;
mdp = md_get_handle();
ASSERT(mdp);
num_nodes = md_node_count(mdp);
ASSERT(num_nodes >= 0);
listsz = num_nodes * sizeof (mde_cookie_t);
listp = (mde_cookie_t *)
(*curr_mach_descrip_memops->meta_allocp)(listsz);
/*
* Import various parameters from the MD. For now,
* the only parameter of interest is whether or not
* domaining features are supported.
*/
init_domaining_capabilities(mdp, listp);
(*curr_mach_descrip_memops->meta_freep)(listp, listsz);
(void) md_fini_handle(mdp);
}
/*
* Simple algorithm for now, grab the global lock and let all
* the clients update themselves in parallel. There is a lot of
* room for improvement here. We could eliminate some scans of
* the DAG by incrementally scanning at lower levels of the DAG
* rather than having each client start its own scan from the root.
*/
void
mdeg_notify_clients(void)
{
md_t *md_new;
mdeg_clnt_t *clnt;
int idx;
int nclnt;
rw_enter(&mdeg.rwlock, RW_READER);
mutex_enter(&mdeg.lock);
/*
* Rotate the MDs
*/
if ((md_new = md_get_handle()) == NULL) {
cmn_err(CE_WARN, "unable to retrieve new MD");
goto done;
}
if (mdeg.md_prev) {
(void) md_fini_handle(mdeg.md_prev);
}
mdeg.md_prev = mdeg.md_curr;
mdeg.md_curr = md_new;
if (mdeg.nclnts == 0) {
MDEG_DBG("mdeg_notify_clients: no clients registered\n");
goto done;
}
/* dispatch the update notification to all clients */
for (idx = 0, nclnt = 0; idx < mdeg.maxclnts; idx++) {
clnt = &mdeg.tbl[idx];
if (!clnt->valid)
continue;
MDEG_DBG("notifying client 0x%lx (%d/%d)\n", clnt->hdl,
++nclnt, mdeg.nclnts);
(void) taskq_dispatch(mdeg.taskq, mdeg_notify_client,
(void *)clnt, TQ_SLEEP);
}
/*
* Wait for all mdeg_notify_client notifications to
* finish while we are still holding mdeg.rwlock.
*/
taskq_wait(mdeg.taskq);
done:
mutex_exit(&mdeg.lock);
rw_exit(&mdeg.rwlock);
}
/*
* Routine used to setup a newly inserted CPU in preparation for starting
* it running code.
*/
int
mp_cpu_configure(int cpuid)
{
md_t *mdp;
mde_cookie_t rootnode, cpunode = MDE_INVAL_ELEM_COOKIE;
int listsz, i;
mde_cookie_t *listp = NULL;
int num_nodes;
uint64_t cpuid_prop;
cpu_t *cpu;
processorid_t id;
ASSERT(MUTEX_HELD(&cpu_lock));
if ((mdp = md_get_handle()) == NULL)
return (ENODEV);
rootnode = md_root_node(mdp);
ASSERT(rootnode != MDE_INVAL_ELEM_COOKIE);
num_nodes = md_node_count(mdp);
ASSERT(num_nodes > 0);
listsz = num_nodes * sizeof (mde_cookie_t);
listp = kmem_zalloc(listsz, KM_SLEEP);
num_nodes = md_scan_dag(mdp, rootnode, md_find_name(mdp, "cpu"),
md_find_name(mdp, "fwd"), listp);
if (num_nodes < 0)
return (ENODEV);
for (i = 0; i < num_nodes; i++) {
if (md_get_prop_val(mdp, listp[i], "id", &cpuid_prop))
break;
if (cpuid_prop == (uint64_t)cpuid) {
cpunode = listp[i];
break;
}
}
if (cpunode == MDE_INVAL_ELEM_COOKIE)
return (ENODEV);
kmem_free(listp, listsz);
mpo_cpu_add(mdp, cpuid);
/*
* Note: uses cpu_lock to protect cpunodes
* which will be modified inside of fill_cpu and
* setup_exec_unit_mappings.
*/
fill_cpu(mdp, cpunode);
/*
* Adding a CPU may cause the execution unit sharing
* relationships to change. Update the mappings in
* the cpunode structures.
*/
setup_chip_mappings(mdp);
setup_exec_unit_mappings(mdp);
/* propagate the updated mappings to the CPU structures */
for (id = 0; id < NCPU; id++) {
if ((cpu = cpu_get(id)) == NULL)
continue;
cpu_map_exec_units(cpu);
}
(void) md_fini_handle(mdp);
if ((i = setup_cpu_common(cpuid)) != 0) {
(void) cleanup_cpu_common(cpuid);
return (i);
}
return (0);
}
/*
* Exported interface to register a LDC endpoint with
* the channel nexus
*/
static int
cnex_reg_chan(dev_info_t *dip, uint64_t id, ldc_dev_t devclass)
{
int idx;
cnex_ldc_t *cldcp;
cnex_ldc_t *new_cldcp;
int listsz, num_nodes, num_channels;
md_t *mdp = NULL;
mde_cookie_t rootnode, *listp = NULL;
uint64_t tmp_id;
uint64_t rxino = (uint64_t)-1;
uint64_t txino = (uint64_t)-1;
cnex_soft_state_t *cnex_ssp;
int status, instance;
dev_info_t *chan_dip = NULL;
/* Get device instance and structure */
instance = ddi_get_instance(dip);
cnex_ssp = ddi_get_soft_state(cnex_state, instance);
/* Check to see if channel is already registered */
mutex_enter(&cnex_ssp->clist_lock);
cldcp = cnex_ssp->clist;
while (cldcp) {
if (cldcp->id == id) {
DWARN("cnex_reg_chan: channel 0x%llx exists\n", id);
mutex_exit(&cnex_ssp->clist_lock);
return (EINVAL);
}
cldcp = cldcp->next;
}
mutex_exit(&cnex_ssp->clist_lock);
/* Get the Tx/Rx inos from the MD */
if ((mdp = md_get_handle()) == NULL) {
DWARN("cnex_reg_chan: cannot init MD\n");
return (ENXIO);
}
num_nodes = md_node_count(mdp);
ASSERT(num_nodes > 0);
listsz = num_nodes * sizeof (mde_cookie_t);
listp = (mde_cookie_t *)kmem_zalloc(listsz, KM_SLEEP);
rootnode = md_root_node(mdp);
/* search for all channel_endpoint nodes */
num_channels = md_scan_dag(mdp, rootnode,
md_find_name(mdp, "channel-endpoint"),
md_find_name(mdp, "fwd"), listp);
if (num_channels <= 0) {
DWARN("cnex_reg_chan: invalid channel id\n");
kmem_free(listp, listsz);
(void) md_fini_handle(mdp);
return (EINVAL);
}
for (idx = 0; idx < num_channels; idx++) {
/* Get the channel ID */
status = md_get_prop_val(mdp, listp[idx], "id", &tmp_id);
if (status) {
DWARN("cnex_reg_chan: cannot read LDC ID\n");
kmem_free(listp, listsz);
(void) md_fini_handle(mdp);
return (ENXIO);
}
if (tmp_id != id)
continue;
/* Get the Tx and Rx ino */
status = md_get_prop_val(mdp, listp[idx], "tx-ino", &txino);
if (status) {
DWARN("cnex_reg_chan: cannot read Tx ino\n");
kmem_free(listp, listsz);
(void) md_fini_handle(mdp);
return (ENXIO);
}
status = md_get_prop_val(mdp, listp[idx], "rx-ino", &rxino);
if (status) {
DWARN("cnex_reg_chan: cannot read Rx ino\n");
kmem_free(listp, listsz);
(void) md_fini_handle(mdp);
return (ENXIO);
}
chan_dip = cnex_find_chan_dip(dip, id, mdp, listp[idx]);
ASSERT(chan_dip != NULL);
}
kmem_free(listp, listsz);
(void) md_fini_handle(mdp);
/*
* check to see if we looped through the list of channel IDs without
* matching one (i.e. an 'ino' has not been initialised).
*/
if ((rxino == -1) || (txino == -1)) {
DERR("cnex_reg_chan: no ID matching '%llx' in MD\n", id);
return (ENOENT);
}
/* Allocate a new channel structure */
new_cldcp = kmem_zalloc(sizeof (*new_cldcp), KM_SLEEP);
/* Initialize the channel */
mutex_init(&new_cldcp->lock, NULL, MUTEX_DRIVER, NULL);
new_cldcp->id = id;
new_cldcp->tx.ino = txino;
new_cldcp->rx.ino = rxino;
new_cldcp->devclass = devclass;
new_cldcp->tx.weight = CNEX_TX_INTR_WEIGHT;
new_cldcp->rx.weight = cnex_class_weight(devclass);
new_cldcp->dip = chan_dip;
/*
* Add channel to nexus channel list.
* Check again to see if channel is already registered since
* clist_lock was dropped above.
*/
mutex_enter(&cnex_ssp->clist_lock);
cldcp = cnex_ssp->clist;
while (cldcp) {
if (cldcp->id == id) {
DWARN("cnex_reg_chan: channel 0x%llx exists\n", id);
mutex_exit(&cnex_ssp->clist_lock);
mutex_destroy(&new_cldcp->lock);
kmem_free(new_cldcp, sizeof (*new_cldcp));
return (EINVAL);
}
cldcp = cldcp->next;
}
new_cldcp->next = cnex_ssp->clist;
cnex_ssp->clist = new_cldcp;
mutex_exit(&cnex_ssp->clist_lock);
return (0);
}
/*
* Obtain an updated MD from the hypervisor and update cpunodes, CPU HW
* sharing data structures, and processor groups.
*/
static void
update_cpu_mappings(void)
{
md_t *mdp;
processorid_t id;
cpu_t *cp;
cpu_pg_t *pgps[NCPU];
if ((mdp = md_get_handle()) == NULL) {
DBG("suspend: md_get_handle failed");
return;
}
DBG("suspend: updating CPU mappings");
mutex_enter(&cpu_lock);
setup_chip_mappings(mdp);
setup_exec_unit_mappings(mdp);
for (id = 0; id < NCPU; id++) {
if ((cp = cpu_get(id)) == NULL)
continue;
cpu_map_exec_units(cp);
}
/*
* Re-calculate processor groups.
*
* First tear down all PG information before adding any new PG
* information derived from the MD we just downloaded. We must
* call pg_cpu_inactive and pg_cpu_active with CPUs paused and
* we want to minimize the number of times pause_cpus is called.
* Inactivating all CPUs would leave PGs without any active CPUs,
* so while CPUs are paused, call pg_cpu_inactive and swap in the
* bootstrap PG structure saving the original PG structure to be
* fini'd afterwards. This prevents the dispatcher from encountering
* PGs in which all CPUs are inactive. Offline CPUs are already
* inactive in their PGs and shouldn't be reactivated, so we must
* not call pg_cpu_inactive or pg_cpu_active for those CPUs.
*/
pause_cpus(NULL);
for (id = 0; id < NCPU; id++) {
if ((cp = cpu_get(id)) == NULL)
continue;
if ((cp->cpu_flags & CPU_OFFLINE) == 0)
pg_cpu_inactive(cp);
pgps[id] = cp->cpu_pg;
pg_cpu_bootstrap(cp);
}
start_cpus();
/*
* pg_cpu_fini* and pg_cpu_init* must be called while CPUs are
* not paused. Use two separate loops here so that we do not
* initialize PG data for CPUs until all the old PG data structures
* are torn down.
*/
for (id = 0; id < NCPU; id++) {
if ((cp = cpu_get(id)) == NULL)
continue;
pg_cpu_fini(cp, pgps[id]);
mpo_cpu_remove(id);
}
/*
* Initialize PG data for each CPU, but leave the bootstrapped
* PG structure in place to avoid running with any PGs containing
* nothing but inactive CPUs.
*/
for (id = 0; id < NCPU; id++) {
if ((cp = cpu_get(id)) == NULL)
continue;
mpo_cpu_add(mdp, id);
pgps[id] = pg_cpu_init(cp, B_TRUE);
}
/*
* Now that PG data has been initialized for all CPUs in the
* system, replace the bootstrapped PG structure with the
* initialized PG structure and call pg_cpu_active for each CPU.
*/
pause_cpus(NULL);
for (id = 0; id < NCPU; id++) {
if ((cp = cpu_get(id)) == NULL)
continue;
cp->cpu_pg = pgps[id];
if ((cp->cpu_flags & CPU_OFFLINE) == 0)
pg_cpu_active(cp);
}
start_cpus();
mutex_exit(&cpu_lock);
(void) md_fini_handle(mdp);
}
/*
* Send a notification to a client immediately after it registers.
* The result_t is a list of all the nodes that match their specified
* nodes of interest, all returned on the added list. This serves
* as a base of reference to the client. All future MD updates are
* relative to this list.
*/
static int
mdeg_notify_client_reg(mdeg_clnt_t *clnt)
{
md_t *mdp = NULL;
mde_str_cookie_t nname;
mde_str_cookie_t aname;
mde_cookie_t startnode;
int nnodes;
int nodechk;
mde_cookie_t *listp = NULL;
mdeg_result_t *mdeg_res = NULL;
int rv = MDEG_SUCCESS;
mutex_enter(&mdeg.lock);
/*
* Handle the special case where the node specification
* is NULL. In this case, call the client callback without
* any results. All processing is left to the client.
*/
if (clnt->pspec == NULL) {
/* call the client callback */
(*clnt->cb)(clnt->cb_arg, NULL);
goto done;
}
if ((mdp = md_get_handle()) == NULL) {
cmn_err(CE_WARN, "unable to retrieve current MD");
rv = MDEG_FAILURE;
goto done;
}
startnode = mdeg_find_start_node(mdp, clnt->pspec);
if (startnode == MDE_INVAL_ELEM_COOKIE) {
/* not much we can do */
cmn_err(CE_WARN, "unable to match node specifier");
rv = MDEG_FAILURE;
goto done;
}
/*
* Use zalloc to provide correct default values for the
* unused removed, match_prev, and match_curr lists.
*/
mdeg_res = kmem_zalloc(sizeof (mdeg_result_t), KM_SLEEP);
nname = md_find_name(mdp, clnt->nmatch->namep);
aname = md_find_name(mdp, "fwd");
nnodes = md_scan_dag(mdp, startnode, nname, aname, NULL);
if (nnodes == 0) {
MDEG_DBG("mdeg_notify_client_reg: no nodes of interest\n");
rv = MDEG_SUCCESS;
goto done;
} else if (nnodes == -1) {
MDEG_DBG("error scanning DAG\n");
rv = MDEG_FAILURE;
goto done;
}
MDEG_DBG("mdeg_notify_client_reg: %d node%s of interest\n",
nnodes, (nnodes == 1) ? "" : "s");
/* get the list of nodes of interest */
listp = kmem_alloc(sizeof (mde_cookie_t) * nnodes, KM_SLEEP);
nodechk = md_scan_dag(mdp, startnode, nname, aname, listp);
ASSERT(nodechk == nnodes);
mdeg_res->added.mdp = mdp;
mdeg_res->added.mdep = listp;
mdeg_res->added.nelem = nnodes;
/* call the client callback */
(*clnt->cb)(clnt->cb_arg, mdeg_res);
done:
mutex_exit(&mdeg.lock);
if (mdp)
(void) md_fini_handle(mdp);
if (listp)
kmem_free(listp, sizeof (mde_cookie_t) * nnodes);
if (mdeg_res)
kmem_free(mdeg_res, sizeof (mdeg_result_t));
return (rv);
}
