static int
bind_virq_to_irq(unsigned int virq, unsigned int cpu)
{
struct evtchn_bind_virq bind_virq;
int evtchn = 0, irq;
mtx_lock_spin(&irq_mapping_update_lock);
if ((irq = pcpu_find(cpu)->pc_virq_to_irq[virq]) == -1) {
if ((irq = find_unbound_irq()) < 0)
goto out;
bind_virq.virq = virq;
bind_virq.vcpu = cpu;
HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
evtchn = bind_virq.port;
evtchn_to_irq[evtchn] = irq;
irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
pcpu_find(cpu)->pc_virq_to_irq[virq] = irq;
bind_evtchn_to_cpu(evtchn, cpu);
}
irq_bindcount[irq]++;
unmask_evtchn(evtchn);
out:
mtx_unlock_spin(&irq_mapping_update_lock);
return irq;
}
/*
* Wait specified idle threads to switch once. This ensures that even
* preempted threads have cycled through the switch function once,
* exiting their codepaths. This allows us to change global pointers
* with no other synchronization.
*/
int
quiesce_cpus(cpuset_t map, const char *wmesg, int prio)
{
struct pcpu *pcpu;
u_int gen[MAXCPU];
int error;
int cpu;
error = 0;
for (cpu = 0; cpu <= mp_maxid; cpu++) {
if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
continue;
pcpu = pcpu_find(cpu);
gen[cpu] = pcpu->pc_idlethread->td_generation;
}
for (cpu = 0; cpu <= mp_maxid; cpu++) {
if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
continue;
pcpu = pcpu_find(cpu);
thread_lock(curthread);
sched_bind(curthread, cpu);
thread_unlock(curthread);
while (gen[cpu] == pcpu->pc_idlethread->td_generation) {
error = tsleep(quiesce_cpus, prio, wmesg, 1);
if (error)
goto out;
}
}
out:
thread_lock(curthread);
sched_unbind(curthread);
thread_unlock(curthread);
return (error);
}
int
bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
{
struct evtchn_bind_ipi bind_ipi;
int irq;
int evtchn = 0;
mtx_lock_spin(&irq_mapping_update_lock);
if ((irq = pcpu_find(cpu)->pc_ipi_to_irq[ipi]) == -1) {
if ((irq = find_unbound_irq()) < 0)
goto out;
bind_ipi.vcpu = cpu;
HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
evtchn = bind_ipi.port;
evtchn_to_irq[evtchn] = irq;
irq_info[irq] = mk_irq_info(IRQT_IPI, ipi, evtchn);
pcpu_find(cpu)->pc_ipi_to_irq[ipi] = irq;
bind_evtchn_to_cpu(evtchn, cpu);
}
irq_bindcount[irq]++;
unmask_evtchn(evtchn);
out:
mtx_unlock_spin(&irq_mapping_update_lock);
return irq;
}
static void
unbind_from_irq(int irq)
{
struct evtchn_close close;
int evtchn = evtchn_from_irq(irq);
int cpu;
mtx_lock_spin(&irq_mapping_update_lock);
if ((--irq_bindcount[irq] == 0) && VALID_EVTCHN(evtchn)) {
close.port = evtchn;
HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
switch (type_from_irq(irq)) {
case IRQT_VIRQ:
cpu = cpu_from_evtchn(evtchn);
pcpu_find(cpu)->pc_virq_to_irq[index_from_irq(irq)] = -1;
break;
case IRQT_IPI:
cpu = cpu_from_evtchn(evtchn);
pcpu_find(cpu)->pc_ipi_to_irq[index_from_irq(irq)] = -1;
break;
default:
break;
}
/* Closed ports are implicitly re-bound to VCPU0. */
bind_evtchn_to_cpu(evtchn, 0);
evtchn_to_irq[evtchn] = -1;
irq_info[irq] = IRQ_UNBOUND;
}
mtx_unlock_spin(&irq_mapping_update_lock);
}
static device_t
cpu_add_child(device_t bus, u_int order, const char *name, int unit)
{
struct cpu_device *cd;
device_t child;
#ifndef __rtems__
struct pcpu *pc;
#endif /* __rtems__ */
if ((cd = malloc(sizeof(*cd), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL)
return (NULL);
resource_list_init(&cd->cd_rl);
#ifndef __rtems__
pc = pcpu_find(device_get_unit(bus));
cd->cd_pcpu = pc;
#endif /* __rtems__ */
child = device_add_child_ordered(bus, order, name, unit);
if (child != NULL) {
#ifndef __rtems__
pc->pc_device = child;
#endif /* __rtems__ */
device_set_ivars(child, cd);
} else
free(cd, M_DEVBUF);
return (child);
}
static struct cpu_group *
chrp_smp_topo(platform_t plat)
{
struct pcpu *pc, *last_pc;
int i, ncores, ncpus;
ncores = ncpus = 0;
last_pc = NULL;
for (i = 0; i <= mp_maxid; i++) {
pc = pcpu_find(i);
if (pc == NULL)
continue;
if (last_pc == NULL || pc->pc_hwref != last_pc->pc_hwref)
ncores++;
last_pc = pc;
ncpus++;
}
if (ncpus % ncores != 0) {
printf("WARNING: Irregular SMP topology. Performance may be "
"suboptimal (%d CPUS, %d cores)\n", ncpus, ncores);
return (smp_topo_none());
}
/* Don't do anything fancier for non-threaded SMP */
if (ncpus == ncores)
return (smp_topo_none());
return (smp_topo_1level(CG_SHARE_L1, ncpus / ncores, CG_FLAG_SMT));
}
/*
* Setup per-CPU domain IDs.
*/
static void
srat_set_cpus(void *dummy)
{
struct cpu_info *cpu;
struct pcpu *pc;
u_int i;
if (srat_physaddr == 0)
return;
for (i = 0; i < MAXCPU; i++) {
if (CPU_ABSENT(i))
continue;
pc = pcpu_find(i);
KASSERT(pc != NULL, ("no pcpu data for CPU %u", i));
cpu = &cpus[pc->pc_apic_id];
if (!cpu->enabled)
panic("SRAT: CPU with APIC ID %u is not known",
pc->pc_apic_id);
pc->pc_domain = cpu->domain;
CPU_SET(i, &cpuset_domain[cpu->domain]);
if (bootverbose)
printf("SRAT: CPU %u has memory domain %d\n", i,
cpu->domain);
}
}
/*
* send an IPI to a set of cpus.
*/
void
ipi_selected(u_int32_t cpus, u_int ipi)
{
struct pcpu *pcpu;
u_int cpuid, new_pending, old_pending;
CTR3(KTR_SMP, "%s: cpus: %x, ipi: %x\n", __func__, cpus, ipi);
while ((cpuid = ffs(cpus)) != 0) {
cpuid--;
cpus &= ~(1 << cpuid);
pcpu = pcpu_find(cpuid);
if (pcpu) {
do {
old_pending = pcpu->pc_pending_ipis;
new_pending = old_pending | ipi;
} while (!atomic_cmpset_int(&pcpu->pc_pending_ipis,
old_pending, new_pending));
if (old_pending)
continue;
mips_ipi_send (cpuid);
}
}
}
/*
* Find the nth present CPU and return its pc_cpuid as well as set the
* pc_acpi_id from the most reliable source.
*/
static int
acpi_pcpu_get_id(uint32_t idx, uint32_t *acpi_id, uint32_t *cpu_id)
{
struct pcpu *pcpu_data;
uint32_t i;
KASSERT(acpi_id != NULL, ("Null acpi_id"));
KASSERT(cpu_id != NULL, ("Null cpu_id"));
for (i = 0; i <= mp_maxid; i++) {
if (CPU_ABSENT(i))
continue;
pcpu_data = pcpu_find(i);
KASSERT(pcpu_data != NULL, ("no pcpu data for %d", i));
if (idx-- == 0) {
/*
* If pc_acpi_id was not initialized (e.g., a non-APIC UP box)
* override it with the value from the ASL. Otherwise, if the
* two don't match, prefer the MADT-derived value. Finally,
* return the pc_cpuid to reference this processor.
*/
if (pcpu_data->pc_acpi_id == 0xffffffff)
pcpu_data->pc_acpi_id = *acpi_id;
else if (pcpu_data->pc_acpi_id != *acpi_id)
*acpi_id = pcpu_data->pc_acpi_id;
*cpu_id = pcpu_data->pc_cpuid;
return (0);
}
}
return (ESRCH);
}
/*
* Send an IPI from the current CPU to the destination CPU.
*/
void
ipi_pcpu(unsigned int cpu, int vector)
{
int irq;
irq = pcpu_find(cpu)->pc_ipi_to_irq[vector];
notify_remote_via_irq(irq);
}
int
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
int trylock, const char *file, int line)
{
if (SCHEDULER_STOPPED())
return (1);
#ifdef INVARIANTS
if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
critical_enter();
KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
curthread) == 0,
("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
rm->lock_object.lo_name, file, line));
critical_exit();
}
#endif
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
("rm_rlock() by idle thread %p on rmlock %s @ %s:%d",
curthread, rm->lock_object.lo_name, file, line));
KASSERT(!rm_destroyed(rm),
("rm_rlock() of destroyed rmlock @ %s:%d", file, line));
if (!trylock) {
KASSERT(!rm_wowned(rm),
("rm_rlock: wlock already held for %s @ %s:%d",
rm->lock_object.lo_name, file, line));
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
NULL);
}
if (_rm_rlock(rm, tracker, trylock)) {
if (trylock)
LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
line);
else
LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
line);
WITNESS_LOCK(&rm->lock_object, 0, file, line);
curthread->td_locks++;
return (1);
} else if (trylock)
LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
return (0);
}
static int
ofw_cpu_attach(device_t dev)
{
struct ofw_cpulist_softc *psc;
struct ofw_cpu_softc *sc;
phandle_t node;
pcell_t cell;
int rv;
sc = device_get_softc(dev);
psc = device_get_softc(device_get_parent(dev));
if (nitems(sc->sc_reg) < psc->sc_addr_cells) {
if (bootverbose)
device_printf(dev, "Too many address cells\n");
return (EINVAL);
}
node = ofw_bus_get_node(dev);
/* Read and validate the reg property for use later */
sc->sc_reg_valid = false;
rv = OF_getencprop(node, "reg", sc->sc_reg, sizeof(sc->sc_reg));
if (rv < 0)
device_printf(dev, "missing 'reg' property\n");
else if ((rv % 4) != 0) {
if (bootverbose)
device_printf(dev, "Malformed reg property\n");
} else if ((rv / 4) != psc->sc_addr_cells) {
if (bootverbose)
device_printf(dev, "Invalid reg size %u\n", rv);
} else
sc->sc_reg_valid = true;
sc->sc_cpu_pcpu = pcpu_find(device_get_unit(dev));
if (OF_getencprop(node, "clock-frequency", &cell, sizeof(cell)) < 0) {
if (bootverbose)
device_printf(dev,
"missing 'clock-frequency' property\n");
} else
sc->sc_nominal_mhz = cell / 1000000; /* convert to MHz */
bus_generic_probe(dev);
return (bus_generic_attach(dev));
}
void
dpcpu_init(void *dpcpu, int cpuid)
{
struct pcpu *pcpu;
pcpu = pcpu_find(cpuid);
pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START;
/*
* Initialize defaults from our linker section.
*/
memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES);
/*
* Place it in the global pcpu offset array.
*/
dpcpu_off[cpuid] = pcpu->pc_dynamic;
}
static uintptr_t
unlock_rm(struct lock_object *lock)
{
struct thread *td;
struct pcpu *pc;
struct rmlock *rm;
struct rm_queue *queue;
struct rm_priotracker *tracker;
uintptr_t how;
rm = (struct rmlock *)lock;
tracker = NULL;
how = 0;
rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
if (rm_wowned(rm))
rm_wunlock(rm);
else {
/*
* Find the right rm_priotracker structure for curthread.
* The guarantee about its uniqueness is given by the fact
* we already asserted the lock wasn't recursively acquired.
*/
critical_enter();
td = curthread;
pc = pcpu_find(curcpu);
for (queue = pc->pc_rm_queue.rmq_next;
queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
tracker = (struct rm_priotracker *)queue;
if ((tracker->rmp_rmlock == rm) &&
(tracker->rmp_thread == td)) {
how = (uintptr_t)tracker;
break;
}
}
KASSERT(tracker != NULL,
("rm_priotracker is non-NULL when lock held in read mode"));
critical_exit();
rm_runlock(rm, tracker);
}
return (how);
}
static void
rm_cleanIPI(void *arg)
{
struct pcpu *pc;
struct rmlock *rm = arg;
struct rm_priotracker *tracker;
struct rm_queue *queue;
pc = pcpu_find(curcpu);
for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
queue = queue->rmq_next) {
tracker = (struct rm_priotracker *)queue;
if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
tracker->rmp_flags = RMPF_ONQUEUE;
mtx_lock_spin(&rm_spinlock);
LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
rmp_qentry);
mtx_unlock_spin(&rm_spinlock);
}
}
}
static int
ofw_cpu_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
{
uint32_t cell;
switch (index) {
case CPU_IVAR_PCPU:
OF_getprop(ofw_bus_get_node(dev), "reg", &cell, sizeof(cell));
*result = (uintptr_t)(pcpu_find(cell));
return (0);
case CPU_IVAR_NOMINAL_MHZ:
cell = 0;
OF_getprop(ofw_bus_get_node(dev), "clock-frequency",
&cell, sizeof(cell));
cell /= 1000000; /* convert to MHz */
*result = (uintptr_t)(cell);
return (0);
}
return (ENOENT);
}
/*
* Setup per-CPU domain IDs from information saved in 'cpus'.
*/
void
acpi_pxm_set_cpu_locality(void)
{
struct cpu_info *cpu;
struct pcpu *pc;
u_int i;
if (srat_physaddr == 0)
return;
for (i = 0; i < MAXCPU; i++) {
if (CPU_ABSENT(i))
continue;
pc = pcpu_find(i);
KASSERT(pc != NULL, ("no pcpu data for CPU %u", i));
cpu = cpu_get_info(pc);
pc->pc_domain = vm_ndomains > 1 ? cpu->domain : 0;
CPU_SET(i, &cpuset_domain[pc->pc_domain]);
if (bootverbose)
printf("SRAT: CPU %u has memory domain %d\n", i,
pc->pc_domain);
}
}
static int
sysctl_kern_cp_times(SYSCTL_HANDLER_ARGS)
{
struct pcpu *pcpu;
int error;
int c;
long *cp_time;
#ifdef SCTL_MASK32
unsigned int cp_time32[CPUSTATES];
int i;
#endif
if (!req->oldptr) {
#ifdef SCTL_MASK32
if (req->flags & SCTL_MASK32)
return SYSCTL_OUT(req, 0, sizeof(cp_time32) * (mp_maxid + 1));
else
#endif
return SYSCTL_OUT(req, 0, sizeof(long) * CPUSTATES * (mp_maxid + 1));
}
for (error = 0, c = 0; error == 0 && c <= mp_maxid; c++) {
if (!CPU_ABSENT(c)) {
pcpu = pcpu_find(c);
cp_time = pcpu->pc_cp_time;
} else {
cp_time = empty;
}
#ifdef SCTL_MASK32
if (req->flags & SCTL_MASK32) {
for (i = 0; i < CPUSTATES; i++)
cp_time32[i] = (unsigned int)cp_time[i];
error = SYSCTL_OUT(req, cp_time32, sizeof(cp_time32));
} else
#endif
error = SYSCTL_OUT(req, cp_time, sizeof(long) * CPUSTATES);
}
return error;
}
static int
ofw_cpu_attach(device_t dev)
{
struct ofw_cpu_softc *sc;
phandle_t node;
uint32_t cell;
sc = device_get_softc(dev);
node = ofw_bus_get_node(dev);
if (OF_getencprop(node, "reg", &cell, sizeof(cell)) < 0) {
cell = device_get_unit(dev);
device_printf(dev, "missing 'reg' property, using %u\n", cell);
}
sc->sc_cpu_pcpu = pcpu_find(cell);
if (OF_getencprop(node, "clock-frequency", &cell, sizeof(cell)) < 0) {
device_printf(dev, "missing 'clock-frequency' property\n");
return (ENXIO);
}
sc->sc_nominal_mhz = cell / 1000000; /* convert to MHz */
bus_generic_probe(dev);
return (bus_generic_attach(dev));
}
static int
acpi_cpu_attach(device_t dev)
{
ACPI_BUFFER buf;
ACPI_OBJECT arg, *obj;
ACPI_OBJECT_LIST arglist;
struct pcpu *pcpu_data;
struct acpi_cpu_softc *sc;
struct acpi_softc *acpi_sc;
ACPI_STATUS status;
u_int features;
int cpu_id, drv_count, i;
driver_t **drivers;
uint32_t cap_set[3];
/* UUID needed by _OSC evaluation */
static uint8_t cpu_oscuuid[16] = { 0x16, 0xA6, 0x77, 0x40, 0x0C, 0x29,
0xBE, 0x47, 0x9E, 0xBD, 0xD8, 0x70,
0x58, 0x71, 0x39, 0x53 };
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
sc = device_get_softc(dev);
sc->cpu_dev = dev;
sc->cpu_handle = acpi_get_handle(dev);
cpu_id = (int)(intptr_t)acpi_get_private(dev);
cpu_softc[cpu_id] = sc;
pcpu_data = pcpu_find(cpu_id);
pcpu_data->pc_device = dev;
sc->cpu_pcpu = pcpu_data;
cpu_smi_cmd = AcpiGbl_FADT.SmiCommand;
cpu_cst_cnt = AcpiGbl_FADT.CstControl;
buf.Pointer = NULL;
buf.Length = ACPI_ALLOCATE_BUFFER;
status = AcpiEvaluateObject(sc->cpu_handle, NULL, NULL, &buf);
if (ACPI_FAILURE(status)) {
device_printf(dev, "attach failed to get Processor obj - %s\n",
AcpiFormatException(status));
return (ENXIO);
}
obj = (ACPI_OBJECT *)buf.Pointer;
sc->cpu_p_blk = obj->Processor.PblkAddress;
sc->cpu_p_blk_len = obj->Processor.PblkLength;
sc->cpu_acpi_id = obj->Processor.ProcId;
AcpiOsFree(obj);
ACPI_DEBUG_PRINT((ACPI_DB_INFO, "acpi_cpu%d: P_BLK at %#x/%d\n",
device_get_unit(dev), sc->cpu_p_blk, sc->cpu_p_blk_len));
/*
* If this is the first cpu we attach, create and initialize the generic
* resources that will be used by all acpi cpu devices.
*/
if (device_get_unit(dev) == 0) {
/* Assume we won't be using generic Cx mode by default */
cpu_cx_generic = FALSE;
/* Install hw.acpi.cpu sysctl tree */
acpi_sc = acpi_device_get_parent_softc(dev);
sysctl_ctx_init(&cpu_sysctl_ctx);
cpu_sysctl_tree = SYSCTL_ADD_NODE(&cpu_sysctl_ctx,
SYSCTL_CHILDREN(acpi_sc->acpi_sysctl_tree), OID_AUTO, "cpu",
CTLFLAG_RD, 0, "node for CPU children");
}
/*
* Before calling any CPU methods, collect child driver feature hints
* and notify ACPI of them. We support unified SMP power control
* so advertise this ourselves. Note this is not the same as independent
* SMP control where each CPU can have different settings.
*/
sc->cpu_features = ACPI_CAP_SMP_SAME | ACPI_CAP_SMP_SAME_C3 |
ACPI_CAP_C1_IO_HALT;
#if defined(__i386__) || defined(__amd64__)
/*
* Ask for MWAIT modes if not disabled and interrupts work
* reasonable with MWAIT.
*/
if (!acpi_disabled("mwait") && cpu_mwait_usable())
sc->cpu_features |= ACPI_CAP_SMP_C1_NATIVE | ACPI_CAP_SMP_C3_NATIVE;
#endif
if (devclass_get_drivers(acpi_cpu_devclass, &drivers, &drv_count) == 0) {
for (i = 0; i < drv_count; i++) {
if (ACPI_GET_FEATURES(drivers[i], &features) == 0)
sc->cpu_features |= features;
}
free(drivers, M_TEMP);
}
/*
* CPU capabilities are specified in
* Intel Processor Vendor-Specific ACPI Interface Specification.
*/
if (sc->cpu_features) {
cap_set[1] = sc->cpu_features;
status = acpi_EvaluateOSC(sc->cpu_handle, cpu_oscuuid, 1, 2, cap_set,
cap_set, false);
if (ACPI_SUCCESS(status)) {
if (cap_set[0] != 0)
device_printf(dev, "_OSC returned status %#x\n", cap_set[0]);
}
else {
arglist.Pointer = &arg;
arglist.Count = 1;
arg.Type = ACPI_TYPE_BUFFER;
arg.Buffer.Length = sizeof(cap_set);
arg.Buffer.Pointer = (uint8_t *)cap_set;
cap_set[0] = 1; /* revision */
cap_set[1] = 1; /* number of capabilities integers */
cap_set[2] = sc->cpu_features;
AcpiEvaluateObject(sc->cpu_handle, "_PDC", &arglist, NULL);
}
}
/* Probe for Cx state support. */
acpi_cpu_cx_probe(sc);
return (0);
}
/* ARGSUSED */
static int
dtrace_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
int flags __unused, struct thread *td)
{
#if __FreeBSD_version < 800039
dtrace_state_t *state = dev->si_drv1;
#else
dtrace_state_t *state;
devfs_get_cdevpriv((void **) &state);
#endif
int error = 0;
if (state == NULL)
return (EINVAL);
if (state->dts_anon) {
ASSERT(dtrace_anon.dta_state == NULL);
state = state->dts_anon;
}
switch (cmd) {
case DTRACEIOC_AGGDESC: {
dtrace_aggdesc_t **paggdesc = (dtrace_aggdesc_t **) addr;
dtrace_aggdesc_t aggdesc;
dtrace_action_t *act;
dtrace_aggregation_t *agg;
int nrecs;
uint32_t offs;
dtrace_recdesc_t *lrec;
void *buf;
size_t size;
uintptr_t dest;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_AGGDESC\n",__func__,__LINE__);
if (copyin((void *) *paggdesc, &aggdesc, sizeof (aggdesc)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
nrecs = aggdesc.dtagd_nrecs;
aggdesc.dtagd_nrecs = 0;
offs = agg->dtag_base;
lrec = &agg->dtag_action.dta_rec;
aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
for (act = agg->dtag_first; ; act = act->dta_next) {
ASSERT(act->dta_intuple ||
DTRACEACT_ISAGG(act->dta_kind));
/*
* If this action has a record size of zero, it
* denotes an argument to the aggregating action.
* Because the presence of this record doesn't (or
* shouldn't) affect the way the data is interpreted,
* we don't copy it out to save user-level the
* confusion of dealing with a zero-length record.
*/
if (act->dta_rec.dtrd_size == 0) {
ASSERT(agg->dtag_hasarg);
continue;
}
aggdesc.dtagd_nrecs++;
if (act == &agg->dtag_action)
break;
}
/*
* Now that we have the size, we need to allocate a temporary
* buffer in which to store the complete description. We need
* the temporary buffer to be able to drop dtrace_lock()
* across the copyout(), below.
*/
size = sizeof (dtrace_aggdesc_t) +
(aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
buf = kmem_alloc(size, KM_SLEEP);
dest = (uintptr_t)buf;
bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
for (act = agg->dtag_first; ; act = act->dta_next) {
dtrace_recdesc_t rec = act->dta_rec;
/*
* See the comment in the above loop for why we pass
* over zero-length records.
*/
if (rec.dtrd_size == 0) {
ASSERT(agg->dtag_hasarg);
continue;
}
if (nrecs-- == 0)
break;
rec.dtrd_offset -= offs;
bcopy(&rec, (void *)dest, sizeof (rec));
dest += sizeof (dtrace_recdesc_t);
if (act == &agg->dtag_action)
break;
}
mutex_exit(&dtrace_lock);
if (copyout(buf, (void *) *paggdesc, dest - (uintptr_t)buf) != 0) {
kmem_free(buf, size);
return (EFAULT);
}
kmem_free(buf, size);
return (0);
}
case DTRACEIOC_AGGSNAP:
case DTRACEIOC_BUFSNAP: {
dtrace_bufdesc_t **pdesc = (dtrace_bufdesc_t **) addr;
dtrace_bufdesc_t desc;
caddr_t cached;
dtrace_buffer_t *buf;
dtrace_debug_output();
if (copyin((void *) *pdesc, &desc, sizeof (desc)) != 0)
return (EFAULT);
DTRACE_IOCTL_PRINTF("%s(%d): %s curcpu %d cpu %d\n",
__func__,__LINE__,
cmd == DTRACEIOC_AGGSNAP ?
"DTRACEIOC_AGGSNAP":"DTRACEIOC_BUFSNAP",
curcpu, desc.dtbd_cpu);
if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
return (ENOENT);
if (pcpu_find(desc.dtbd_cpu) == NULL)
return (ENOENT);
mutex_enter(&dtrace_lock);
if (cmd == DTRACEIOC_BUFSNAP) {
buf = &state->dts_buffer[desc.dtbd_cpu];
} else {
buf = &state->dts_aggbuffer[desc.dtbd_cpu];
}
if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
size_t sz = buf->dtb_offset;
if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
mutex_exit(&dtrace_lock);
return (EBUSY);
}
/*
* If this buffer has already been consumed, we're
* going to indicate that there's nothing left here
* to consume.
*/
if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
mutex_exit(&dtrace_lock);
desc.dtbd_size = 0;
desc.dtbd_drops = 0;
desc.dtbd_errors = 0;
desc.dtbd_oldest = 0;
sz = sizeof (desc);
if (copyout(&desc, (void *) *pdesc, sz) != 0)
return (EFAULT);
return (0);
}
/*
* If this is a ring buffer that has wrapped, we want
* to copy the whole thing out.
*/
if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
dtrace_buffer_polish(buf);
sz = buf->dtb_size;
}
if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
mutex_exit(&dtrace_lock);
return (EFAULT);
}
desc.dtbd_size = sz;
desc.dtbd_drops = buf->dtb_drops;
desc.dtbd_errors = buf->dtb_errors;
desc.dtbd_oldest = buf->dtb_xamot_offset;
mutex_exit(&dtrace_lock);
if (copyout(&desc, (void *) *pdesc, sizeof (desc)) != 0)
return (EFAULT);
buf->dtb_flags |= DTRACEBUF_CONSUMED;
return (0);
}
if (buf->dtb_tomax == NULL) {
ASSERT(buf->dtb_xamot == NULL);
mutex_exit(&dtrace_lock);
return (ENOENT);
}
cached = buf->dtb_tomax;
ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
dtrace_xcall(desc.dtbd_cpu,
(dtrace_xcall_t)dtrace_buffer_switch, buf);
state->dts_errors += buf->dtb_xamot_errors;
/*
* If the buffers did not actually switch, then the cross call
* did not take place -- presumably because the given CPU is
* not in the ready set. If this is the case, we'll return
* ENOENT.
*/
if (buf->dtb_tomax == cached) {
ASSERT(buf->dtb_xamot != cached);
mutex_exit(&dtrace_lock);
return (ENOENT);
}
ASSERT(cached == buf->dtb_xamot);
DTRACE_IOCTL_PRINTF("%s(%d): copyout the buffer snapshot\n",__func__,__LINE__);
/*
* We have our snapshot; now copy it out.
*/
if (copyout(buf->dtb_xamot, desc.dtbd_data,
buf->dtb_xamot_offset) != 0) {
mutex_exit(&dtrace_lock);
return (EFAULT);
}
desc.dtbd_size = buf->dtb_xamot_offset;
desc.dtbd_drops = buf->dtb_xamot_drops;
desc.dtbd_errors = buf->dtb_xamot_errors;
desc.dtbd_oldest = 0;
mutex_exit(&dtrace_lock);
DTRACE_IOCTL_PRINTF("%s(%d): copyout buffer desc: size %zd drops %lu errors %lu\n",__func__,__LINE__,(size_t) desc.dtbd_size,(u_long) desc.dtbd_drops,(u_long) desc.dtbd_errors);
/*
* Finally, copy out the buffer description.
*/
if (copyout(&desc, (void *) *pdesc, sizeof (desc)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_CONF: {
dtrace_conf_t conf;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_CONF\n",__func__,__LINE__);
bzero(&conf, sizeof (conf));
conf.dtc_difversion = DIF_VERSION;
conf.dtc_difintregs = DIF_DIR_NREGS;
conf.dtc_diftupregs = DIF_DTR_NREGS;
conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
*((dtrace_conf_t *) addr) = conf;
return (0);
}
case DTRACEIOC_DOFGET: {
dof_hdr_t **pdof = (dof_hdr_t **) addr;
dof_hdr_t hdr, *dof = *pdof;
int rval;
uint64_t len;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_DOFGET\n",__func__,__LINE__);
if (copyin((void *)dof, &hdr, sizeof (hdr)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
dof = dtrace_dof_create(state);
mutex_exit(&dtrace_lock);
len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
rval = copyout(dof, (void *) *pdof, len);
dtrace_dof_destroy(dof);
return (rval == 0 ? 0 : EFAULT);
}
case DTRACEIOC_ENABLE: {
dof_hdr_t *dof = NULL;
dtrace_enabling_t *enab = NULL;
dtrace_vstate_t *vstate;
int err = 0;
int rval;
dtrace_enable_io_t *p = (dtrace_enable_io_t *) addr;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_ENABLE\n",__func__,__LINE__);
/*
* If a NULL argument has been passed, we take this as our
* cue to reevaluate our enablings.
*/
if (p->dof == NULL) {
dtrace_enabling_matchall();
return (0);
}
if ((dof = dtrace_dof_copyin((uintptr_t) p->dof, &rval)) == NULL)
return (EINVAL);
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
vstate = &state->dts_vstate;
if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
dtrace_dof_destroy(dof);
return (EBUSY);
}
if (dtrace_dof_slurp(dof, vstate, td->td_ucred, &enab, 0, B_TRUE) != 0) {
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
dtrace_dof_destroy(dof);
return (EINVAL);
}
if ((rval = dtrace_dof_options(dof, state)) != 0) {
dtrace_enabling_destroy(enab);
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
dtrace_dof_destroy(dof);
return (rval);
}
if ((err = dtrace_enabling_match(enab, &p->n_matched)) == 0) {
err = dtrace_enabling_retain(enab);
} else {
dtrace_enabling_destroy(enab);
}
mutex_exit(&cpu_lock);
mutex_exit(&dtrace_lock);
dtrace_dof_destroy(dof);
return (err);
}
case DTRACEIOC_EPROBE: {
dtrace_eprobedesc_t **pepdesc = (dtrace_eprobedesc_t **) addr;
dtrace_eprobedesc_t epdesc;
dtrace_ecb_t *ecb;
dtrace_action_t *act;
void *buf;
size_t size;
uintptr_t dest;
int nrecs;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_EPROBE\n",__func__,__LINE__);
if (copyin((void *)*pepdesc, &epdesc, sizeof (epdesc)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
if (ecb->dte_probe == NULL) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
epdesc.dtepd_uarg = ecb->dte_uarg;
epdesc.dtepd_size = ecb->dte_size;
nrecs = epdesc.dtepd_nrecs;
epdesc.dtepd_nrecs = 0;
for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
continue;
epdesc.dtepd_nrecs++;
}
/*
* Now that we have the size, we need to allocate a temporary
* buffer in which to store the complete description. We need
* the temporary buffer to be able to drop dtrace_lock()
* across the copyout(), below.
*/
size = sizeof (dtrace_eprobedesc_t) +
(epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
buf = kmem_alloc(size, KM_SLEEP);
dest = (uintptr_t)buf;
bcopy(&epdesc, (void *)dest, sizeof (epdesc));
dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
continue;
if (nrecs-- == 0)
break;
bcopy(&act->dta_rec, (void *)dest,
sizeof (dtrace_recdesc_t));
dest += sizeof (dtrace_recdesc_t);
}
mutex_exit(&dtrace_lock);
if (copyout(buf, (void *) *pepdesc, dest - (uintptr_t)buf) != 0) {
kmem_free(buf, size);
return (EFAULT);
}
kmem_free(buf, size);
return (0);
}
case DTRACEIOC_FORMAT: {
dtrace_fmtdesc_t *fmt = (dtrace_fmtdesc_t *) addr;
char *str;
int len;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_FORMAT\n",__func__,__LINE__);
mutex_enter(&dtrace_lock);
if (fmt->dtfd_format == 0 ||
fmt->dtfd_format > state->dts_nformats) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
/*
* Format strings are allocated contiguously and they are
* never freed; if a format index is less than the number
* of formats, we can assert that the format map is non-NULL
* and that the format for the specified index is non-NULL.
*/
ASSERT(state->dts_formats != NULL);
str = state->dts_formats[fmt->dtfd_format - 1];
ASSERT(str != NULL);
len = strlen(str) + 1;
if (len > fmt->dtfd_length) {
fmt->dtfd_length = len;
} else {
if (copyout(str, fmt->dtfd_string, len) != 0) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
}
mutex_exit(&dtrace_lock);
return (0);
}
case DTRACEIOC_GO: {
int rval;
processorid_t *cpuid = (processorid_t *) addr;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_GO\n",__func__,__LINE__);
rval = dtrace_state_go(state, cpuid);
return (rval);
}
case DTRACEIOC_PROBEARG: {
dtrace_argdesc_t *desc = (dtrace_argdesc_t *) addr;
dtrace_probe_t *probe;
dtrace_provider_t *prov;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_PROBEARG\n",__func__,__LINE__);
if (desc->dtargd_id == DTRACE_IDNONE)
return (EINVAL);
if (desc->dtargd_ndx == DTRACE_ARGNONE)
return (EINVAL);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&mod_lock);
mutex_enter(&dtrace_lock);
if (desc->dtargd_id > dtrace_nprobes) {
mutex_exit(&dtrace_lock);
mutex_exit(&mod_lock);
mutex_exit(&dtrace_provider_lock);
return (EINVAL);
}
if ((probe = dtrace_probes[desc->dtargd_id - 1]) == NULL) {
mutex_exit(&dtrace_lock);
mutex_exit(&mod_lock);
mutex_exit(&dtrace_provider_lock);
return (EINVAL);
}
mutex_exit(&dtrace_lock);
prov = probe->dtpr_provider;
if (prov->dtpv_pops.dtps_getargdesc == NULL) {
/*
* There isn't any typed information for this probe.
* Set the argument number to DTRACE_ARGNONE.
*/
desc->dtargd_ndx = DTRACE_ARGNONE;
} else {
desc->dtargd_native[0] = '\0';
desc->dtargd_xlate[0] = '\0';
desc->dtargd_mapping = desc->dtargd_ndx;
prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
probe->dtpr_id, probe->dtpr_arg, desc);
}
mutex_exit(&mod_lock);
mutex_exit(&dtrace_provider_lock);
return (0);
}
case DTRACEIOC_PROBEMATCH:
case DTRACEIOC_PROBES: {
dtrace_probedesc_t *p_desc = (dtrace_probedesc_t *) addr;
dtrace_probe_t *probe = NULL;
dtrace_probekey_t pkey;
dtrace_id_t i;
int m = 0;
uint32_t priv = 0;
uid_t uid = 0;
zoneid_t zoneid = 0;
DTRACE_IOCTL_PRINTF("%s(%d): %s\n",__func__,__LINE__,
cmd == DTRACEIOC_PROBEMATCH ?
"DTRACEIOC_PROBEMATCH":"DTRACEIOC_PROBES");
p_desc->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
p_desc->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
p_desc->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
p_desc->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
/*
* Before we attempt to match this probe, we want to give
* all providers the opportunity to provide it.
*/
if (p_desc->dtpd_id == DTRACE_IDNONE) {
mutex_enter(&dtrace_provider_lock);
dtrace_probe_provide(p_desc, NULL);
mutex_exit(&dtrace_provider_lock);
p_desc->dtpd_id++;
}
if (cmd == DTRACEIOC_PROBEMATCH) {
dtrace_probekey(p_desc, &pkey);
pkey.dtpk_id = DTRACE_IDNONE;
}
dtrace_cred2priv(td->td_ucred, &priv, &uid, &zoneid);
mutex_enter(&dtrace_lock);
if (cmd == DTRACEIOC_PROBEMATCH) {
for (i = p_desc->dtpd_id; i <= dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i - 1]) != NULL &&
(m = dtrace_match_probe(probe, &pkey,
priv, uid, zoneid)) != 0)
break;
}
if (m < 0) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
} else {
for (i = p_desc->dtpd_id; i <= dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i - 1]) != NULL &&
dtrace_match_priv(probe, priv, uid, zoneid))
break;
}
}
if (probe == NULL) {
mutex_exit(&dtrace_lock);
return (ESRCH);
}
dtrace_probe_description(probe, p_desc);
mutex_exit(&dtrace_lock);
return (0);
}
case DTRACEIOC_PROVIDER: {
dtrace_providerdesc_t *pvd = (dtrace_providerdesc_t *) addr;
dtrace_provider_t *pvp;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_PROVIDER\n",__func__,__LINE__);
pvd->dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
mutex_enter(&dtrace_provider_lock);
for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
if (strcmp(pvp->dtpv_name, pvd->dtvd_name) == 0)
break;
}
mutex_exit(&dtrace_provider_lock);
if (pvp == NULL)
return (ESRCH);
bcopy(&pvp->dtpv_priv, &pvd->dtvd_priv, sizeof (dtrace_ppriv_t));
bcopy(&pvp->dtpv_attr, &pvd->dtvd_attr, sizeof (dtrace_pattr_t));
return (0);
}
case DTRACEIOC_REPLICATE: {
dtrace_repldesc_t *desc = (dtrace_repldesc_t *) addr;
dtrace_probedesc_t *match = &desc->dtrpd_match;
dtrace_probedesc_t *create = &desc->dtrpd_create;
int err;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_REPLICATE\n",__func__,__LINE__);
match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
mutex_enter(&dtrace_lock);
err = dtrace_enabling_replicate(state, match, create);
mutex_exit(&dtrace_lock);
return (err);
}
case DTRACEIOC_STATUS: {
dtrace_status_t *stat = (dtrace_status_t *) addr;
dtrace_dstate_t *dstate;
int i, j;
uint64_t nerrs;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_STATUS\n",__func__,__LINE__);
/*
* See the comment in dtrace_state_deadman() for the reason
* for setting dts_laststatus to INT64_MAX before setting
* it to the correct value.
*/
state->dts_laststatus = INT64_MAX;
dtrace_membar_producer();
state->dts_laststatus = dtrace_gethrtime();
bzero(stat, sizeof (*stat));
mutex_enter(&dtrace_lock);
if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
mutex_exit(&dtrace_lock);
return (ENOENT);
}
if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
stat->dtst_exiting = 1;
nerrs = state->dts_errors;
dstate = &state->dts_vstate.dtvs_dynvars;
for (i = 0; i < NCPU; i++) {
#if !defined(sun)
if (pcpu_find(i) == NULL)
continue;
#endif
dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
stat->dtst_dyndrops += dcpu->dtdsc_drops;
stat->dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
stat->dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
stat->dtst_filled++;
nerrs += state->dts_buffer[i].dtb_errors;
for (j = 0; j < state->dts_nspeculations; j++) {
dtrace_speculation_t *spec;
dtrace_buffer_t *buf;
spec = &state->dts_speculations[j];
buf = &spec->dtsp_buffer[i];
stat->dtst_specdrops += buf->dtb_xamot_drops;
}
}
stat->dtst_specdrops_busy = state->dts_speculations_busy;
stat->dtst_specdrops_unavail = state->dts_speculations_unavail;
stat->dtst_stkstroverflows = state->dts_stkstroverflows;
stat->dtst_dblerrors = state->dts_dblerrors;
stat->dtst_killed =
(state->dts_activity == DTRACE_ACTIVITY_KILLED);
stat->dtst_errors = nerrs;
mutex_exit(&dtrace_lock);
return (0);
}
case DTRACEIOC_STOP: {
int rval;
processorid_t *cpuid = (processorid_t *) addr;
DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_STOP\n",__func__,__LINE__);
mutex_enter(&dtrace_lock);
rval = dtrace_state_stop(state, cpuid);
mutex_exit(&dtrace_lock);
return (rval);
}
default:
error = ENOTTY;
}
return (error);
}
int
acpi_sleep_machdep(struct acpi_softc *sc, int state)
{
ACPI_STATUS status;
struct pcb *pcb;
#ifdef __amd64__
struct pcpu *pc;
int i;
#endif
if (sc->acpi_wakeaddr == 0ul)
return (-1); /* couldn't alloc wake memory */
#ifdef SMP
suspcpus = all_cpus;
CPU_CLR(PCPU_GET(cpuid), &suspcpus);
#endif
if (acpi_resume_beep != 0)
timer_spkr_acquire();
AcpiSetFirmwareWakingVector(sc->acpi_wakephys, 0);
intr_suspend();
pcb = &susppcbs[0]->sp_pcb;
if (savectx(pcb)) {
#ifdef __amd64__
fpususpend(susppcbs[0]->sp_fpususpend);
#else
npxsuspend(susppcbs[0]->sp_fpususpend);
#endif
#ifdef SMP
if (!CPU_EMPTY(&suspcpus) && suspend_cpus(suspcpus) == 0) {
device_printf(sc->acpi_dev, "Failed to suspend APs\n");
return (0); /* couldn't sleep */
}
#endif
#ifdef __amd64__
hw_ibrs_active = 0;
hw_ssb_active = 0;
cpu_stdext_feature3 = 0;
CPU_FOREACH(i) {
pc = pcpu_find(i);
pc->pc_ibpb_set = 0;
}
#endif
WAKECODE_FIXUP(resume_beep, uint8_t, (acpi_resume_beep != 0));
WAKECODE_FIXUP(reset_video, uint8_t, (acpi_reset_video != 0));
#ifdef __amd64__
WAKECODE_FIXUP(wakeup_efer, uint64_t, rdmsr(MSR_EFER) &
~(EFER_LMA));
#else
if ((amd_feature & AMDID_NX) != 0)
WAKECODE_FIXUP(wakeup_efer, uint64_t, rdmsr(MSR_EFER));
WAKECODE_FIXUP(wakeup_cr4, register_t, pcb->pcb_cr4);
#endif
WAKECODE_FIXUP(wakeup_pcb, struct pcb *, pcb);
WAKECODE_FIXUP(wakeup_gdt, uint16_t, pcb->pcb_gdt.rd_limit);
WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t, pcb->pcb_gdt.rd_base);
#ifdef __i386__
/*
* Map some low memory with virt == phys for ACPI wakecode
* to use to jump to high memory after enabling paging. This
* is the same as for similar jump in locore, except the
* jump is a single instruction, and we know its address
* more precisely so only need a single PTD, and we have to
* be careful to use the kernel map (PTD[0] is for curthread
* which may be a user thread in deprecated APIs).
*/
pmap_remap_lowptdi(true);
#endif
/* Call ACPICA to enter the desired sleep state */
if (state == ACPI_STATE_S4 && sc->acpi_s4bios)
status = AcpiEnterSleepStateS4bios();
else
status = AcpiEnterSleepState(state);
if (ACPI_FAILURE(status)) {
device_printf(sc->acpi_dev,
"AcpiEnterSleepState failed - %s\n",
AcpiFormatException(status));
return (0); /* couldn't sleep */
}
if (acpi_susp_bounce)
resumectx(pcb);
for (;;)
ia32_pause();
} else {
static int
acpi_cpu_attach(device_t dev)
{
ACPI_BUFFER buf;
ACPI_OBJECT arg[4], *obj;
ACPI_OBJECT_LIST arglist;
struct pcpu *pcpu_data;
struct acpi_cpu_softc *sc;
struct acpi_softc *acpi_sc;
ACPI_STATUS status;
u_int features;
int cpu_id, drv_count, i;
driver_t **drivers;
uint32_t cap_set[3];
/* UUID needed by _OSC evaluation */
static uint8_t cpu_oscuuid[16] = { 0x16, 0xA6, 0x77, 0x40, 0x0C, 0x29,
0xBE, 0x47, 0x9E, 0xBD, 0xD8, 0x70,
0x58, 0x71, 0x39, 0x53 };
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
sc = device_get_softc(dev);
sc->cpu_dev = dev;
sc->cpu_handle = acpi_get_handle(dev);
cpu_id = acpi_get_magic(dev);
cpu_softc[cpu_id] = sc;
pcpu_data = pcpu_find(cpu_id);
pcpu_data->pc_device = dev;
sc->cpu_pcpu = pcpu_data;
cpu_smi_cmd = AcpiGbl_FADT.SmiCommand;
cpu_cst_cnt = AcpiGbl_FADT.CstControl;
buf.Pointer = NULL;
buf.Length = ACPI_ALLOCATE_BUFFER;
status = AcpiEvaluateObject(sc->cpu_handle, NULL, NULL, &buf);
if (ACPI_FAILURE(status)) {
device_printf(dev, "attach failed to get Processor obj - %s\n",
AcpiFormatException(status));
return (ENXIO);
}
obj = (ACPI_OBJECT *)buf.Pointer;
sc->cpu_p_blk = obj->Processor.PblkAddress;
sc->cpu_p_blk_len = obj->Processor.PblkLength;
sc->cpu_acpi_id = obj->Processor.ProcId;
AcpiOsFree(obj);
ACPI_DEBUG_PRINT((ACPI_DB_INFO, "acpi_cpu%d: P_BLK at %#x/%d\n",
device_get_unit(dev), sc->cpu_p_blk, sc->cpu_p_blk_len));
/*
* If this is the first cpu we attach, create and initialize the generic
* resources that will be used by all acpi cpu devices.
*/
if (device_get_unit(dev) == 0) {
/* Assume we won't be using generic Cx mode by default */
cpu_cx_generic = FALSE;
/* Install hw.acpi.cpu sysctl tree */
acpi_sc = acpi_device_get_parent_softc(dev);
sysctl_ctx_init(&cpu_sysctl_ctx);
cpu_sysctl_tree = SYSCTL_ADD_NODE(&cpu_sysctl_ctx,
SYSCTL_CHILDREN(acpi_sc->acpi_sysctl_tree), OID_AUTO, "cpu",
CTLFLAG_RD, 0, "node for CPU children");
/* Queue post cpu-probing task handler */
AcpiOsExecute(OSL_NOTIFY_HANDLER, acpi_cpu_startup, NULL);
}
/*
* Before calling any CPU methods, collect child driver feature hints
* and notify ACPI of them. We support unified SMP power control
* so advertise this ourselves. Note this is not the same as independent
* SMP control where each CPU can have different settings.
*/
sc->cpu_features = ACPI_CAP_SMP_SAME | ACPI_CAP_SMP_SAME_C3;
if (devclass_get_drivers(acpi_cpu_devclass, &drivers, &drv_count) == 0) {
for (i = 0; i < drv_count; i++) {
if (ACPI_GET_FEATURES(drivers[i], &features) == 0)
sc->cpu_features |= features;
}
free(drivers, M_TEMP);
}
/*
* CPU capabilities are specified as a buffer of 32-bit integers:
* revision, count, and one or more capabilities. The revision of
* "1" is not specified anywhere but seems to match Linux.
*/
if (sc->cpu_features) {
arglist.Pointer = arg;
arglist.Count = 1;
arg[0].Type = ACPI_TYPE_BUFFER;
arg[0].Buffer.Length = sizeof(cap_set);
arg[0].Buffer.Pointer = (uint8_t *)cap_set;
cap_set[0] = 1; /* revision */
cap_set[1] = 1; /* number of capabilities integers */
cap_set[2] = sc->cpu_features;
AcpiEvaluateObject(sc->cpu_handle, "_PDC", &arglist, NULL);
/*
* On some systems we need to evaluate _OSC so that the ASL
* loads the _PSS and/or _PDC methods at runtime.
*
* TODO: evaluate failure of _OSC.
*/
arglist.Pointer = arg;
arglist.Count = 4;
arg[0].Type = ACPI_TYPE_BUFFER;
arg[0].Buffer.Length = sizeof(cpu_oscuuid);
arg[0].Buffer.Pointer = cpu_oscuuid; /* UUID */
arg[1].Type = ACPI_TYPE_INTEGER;
arg[1].Integer.Value = 1; /* revision */
arg[2].Type = ACPI_TYPE_INTEGER;
arg[2].Integer.Value = 1; /* count */
arg[3].Type = ACPI_TYPE_BUFFER;
arg[3].Buffer.Length = sizeof(cap_set); /* Capabilities buffer */
arg[3].Buffer.Pointer = (uint8_t *)cap_set;
cap_set[0] = 0;
AcpiEvaluateObject(sc->cpu_handle, "_OSC", &arglist, NULL);
}
/* Probe for Cx state support. */
acpi_cpu_cx_probe(sc);
/* Finally, call identify and probe/attach for child devices. */
bus_generic_probe(dev);
bus_generic_attach(dev);
return (0);
}
/*
* Note that this does not need to use witness_assert() for read lock
* assertions since an exact count of read locks held by this thread
* is computable.
*/
void
_rm_assert(const struct rmlock *rm, int what, const char *file, int line)
{
int count;
if (panicstr != NULL)
return;
switch (what) {
case RA_LOCKED:
case RA_LOCKED | RA_RECURSED:
case RA_LOCKED | RA_NOTRECURSED:
case RA_RLOCKED:
case RA_RLOCKED | RA_RECURSED:
case RA_RLOCKED | RA_NOTRECURSED:
/*
* Handle the write-locked case. Unlike other
* primitives, writers can never recurse.
*/
if (rm_wowned(rm)) {
if (what & RA_RLOCKED)
panic("Lock %s exclusively locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
if (what & RA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
}
critical_enter();
count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
critical_exit();
if (count == 0)
panic("Lock %s not %slocked @ %s:%d\n",
rm->lock_object.lo_name, (what & RA_RLOCKED) ?
"read " : "", file, line);
if (count > 1) {
if (what & RA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
rm->lock_object.lo_name, file, line);
} else if (what & RA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
case RA_WLOCKED:
if (!rm_wowned(rm))
panic("Lock %s not exclusively locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
case RA_UNLOCKED:
if (rm_wowned(rm))
panic("Lock %s exclusively locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
critical_enter();
count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
critical_exit();
if (count != 0)
panic("Lock %s read locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
default:
panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
line);
}
}
static int
_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
{
struct pcpu *pc;
critical_enter();
pc = pcpu_find(curcpu);
/* Check if we just need to do a proper critical_exit. */
if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
critical_exit();
return (1);
}
/* Remove our tracker from the per-cpu list. */
rm_tracker_remove(pc, tracker);
/* Check to see if the IPI granted us the lock after all. */
if (tracker->rmp_flags) {
/* Just add back tracker - we hold the lock. */
rm_tracker_add(pc, tracker);
critical_exit();
return (1);
}
/*
* We allow readers to acquire a lock even if a writer is blocked if
* the lock is recursive and the reader already holds the lock.
*/
if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
/*
* Just grant the lock if this thread already has a tracker
* for this lock on the per-cpu queue.
*/
if (rm_trackers_present(pc, rm, curthread) != 0) {
mtx_lock_spin(&rm_spinlock);
LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
rmp_qentry);
tracker->rmp_flags = RMPF_ONQUEUE;
mtx_unlock_spin(&rm_spinlock);
rm_tracker_add(pc, tracker);
critical_exit();
return (1);
}
}
sched_unpin();
critical_exit();
if (trylock) {
if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
if (!sx_try_xlock(&rm->rm_lock_sx))
return (0);
} else {
if (!mtx_trylock(&rm->rm_lock_mtx))
return (0);
}
} else {
if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
THREAD_SLEEPING_OK();
sx_xlock(&rm->rm_lock_sx);
THREAD_NO_SLEEPING();
} else
mtx_lock(&rm->rm_lock_mtx);
}
critical_enter();
pc = pcpu_find(curcpu);
CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
rm_tracker_add(pc, tracker);
sched_pin();
critical_exit();
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
sx_xunlock(&rm->rm_lock_sx);
else
mtx_unlock(&rm->rm_lock_mtx);
return (1);
}
