/*
* Called from lwp_exit() and thread_exit()
*/
void
kcpc_passivate(void)
{
kcpc_ctx_t *ctx = curthread->t_cpc_ctx;
kcpc_set_t *set = curthread->t_cpc_set;
if (set == NULL)
return;
/*
* We're cleaning up after this thread; ensure there are no dangling
* CPC pointers left behind. The context and set will be freed by
* freectx() in the case of an LWP-bound set, and by kcpc_unbind() in
* the case of a CPU-bound set.
*/
curthread->t_cpc_ctx = NULL;
if (ctx == NULL) {
/*
* This thread has a set but no context; it must be a CPU-bound
* set. The hardware will be stopped via kcpc_unbind() when the
* process exits and closes its file descriptors with
* kcpc_close(). Our only job here is to clean up this thread's
* state; the set will be freed with the unbind().
*/
(void) kcpc_unbind(set);
/*
* Unbinding a set belonging to the current thread should clear
* its set pointer.
*/
ASSERT(curthread->t_cpc_set == NULL);
return;
}
curthread->t_cpc_set = NULL;
/*
* This thread/LWP is exiting but context switches will continue to
* happen for a bit as the exit proceeds. Kernel preemption must be
* disabled here to prevent a race between checking or setting the
* INVALID_STOPPED flag here and kcpc_restore() setting the flag during
* a context switch.
*/
kpreempt_disable();
if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) {
pcbe_ops->pcbe_allstop();
atomic_or_uint(&ctx->kc_flags,
KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);
}
kpreempt_enable();
}
/*
* System call to access CPU performance counters.
*/
static int
cpc(int cmd, id_t lwpid, void *udata1, void *udata2, void *udata3)
{
kthread_t *t;
int error;
int size;
const char *str;
int code;
/*
* This CPC syscall should only be loaded if it found a PCBE to use.
*/
ASSERT(pcbe_ops != NULL);
if (curproc->p_agenttp == curthread) {
/*
* Only if /proc is invoking this system call from
* the agent thread do we allow the caller to examine
* the contexts of other lwps in the process. And
* because we know we're the agent, we know we don't
* have to grab p_lock because no-one else can change
* the state of the process.
*/
if ((t = idtot(curproc, lwpid)) == NULL || t == curthread)
return (set_errno(ESRCH));
ASSERT(t->t_tid == lwpid && ttolwp(t) != NULL);
} else
t = curthread;
if (t->t_cpc_set == NULL && (cmd == CPC_SAMPLE || cmd == CPC_RELE))
return (set_errno(EINVAL));
switch (cmd) {
case CPC_BIND:
/*
* udata1 = pointer to packed nvlist buffer
* udata2 = size of packed nvlist buffer
* udata3 = User addr to return error subcode in.
*/
rw_enter(&kcpc_cpuctx_lock, RW_READER);
if (kcpc_cpuctx || dtrace_cpc_in_use) {
rw_exit(&kcpc_cpuctx_lock);
return (set_errno(EAGAIN));
}
if (kcpc_hw_lwp_hook() != 0) {
rw_exit(&kcpc_cpuctx_lock);
return (set_errno(EACCES));
}
/*
* An LWP may only have one set bound to it at a time; if there
* is a set bound to this LWP already, we unbind it here.
*/
if (t->t_cpc_set != NULL)
(void) kcpc_unbind(t->t_cpc_set);
ASSERT(t->t_cpc_set == NULL);
if ((error = kcpc_copyin_set(&t->t_cpc_set, udata1,
(size_t)udata2)) != 0) {
rw_exit(&kcpc_cpuctx_lock);
return (set_errno(error));
}
if ((error = kcpc_verify_set(t->t_cpc_set)) != 0) {
rw_exit(&kcpc_cpuctx_lock);
kcpc_free_set(t->t_cpc_set);
t->t_cpc_set = NULL;
if (copyout(&error, udata3, sizeof (error)) == -1)
return (set_errno(EFAULT));
return (set_errno(EINVAL));
}
if ((error = kcpc_bind_thread(t->t_cpc_set, t, &code)) != 0) {
rw_exit(&kcpc_cpuctx_lock);
kcpc_free_set(t->t_cpc_set);
t->t_cpc_set = NULL;
/*
* EINVAL and EACCES are the only errors with more
* specific subcodes.
*/
if ((error == EINVAL || error == EACCES) &&
copyout(&code, udata3, sizeof (code)) == -1)
return (set_errno(EFAULT));
return (set_errno(error));
}
rw_exit(&kcpc_cpuctx_lock);
return (0);
case CPC_SAMPLE:
/*
* udata1 = pointer to user's buffer
* udata2 = pointer to user's hrtime
* udata3 = pointer to user's tick
*/
/*
* We only allow thread-bound sets to be sampled via the
* syscall, so if this set has a CPU-bound context, return an
* error.
*/
if (t->t_cpc_set->ks_ctx->kc_cpuid != -1)
return (set_errno(EINVAL));
if ((error = kcpc_sample(t->t_cpc_set, udata1, udata2,
udata3)) != 0)
return (set_errno(error));
return (0);
case CPC_PRESET:
case CPC_RESTART:
/*
* These are valid only if this lwp has a bound set.
*/
if (t->t_cpc_set == NULL)
return (set_errno(EINVAL));
if (cmd == CPC_PRESET) {
/*
* The preset is shipped up to us from userland in two
* parts. This lets us handle 64-bit values from 32-bit
* and 64-bit applications in the same manner.
*
* udata1 = index of request to preset
* udata2 = new 64-bit preset (most sig. 32 bits)
* udata3 = new 64-bit preset (least sig. 32 bits)
*/
if ((error = kcpc_preset(t->t_cpc_set, (intptr_t)udata1,
((uint64_t)(uintptr_t)udata2 << 32ULL) |
(uint64_t)(uintptr_t)udata3)) != 0)
return (set_errno(error));
} else {
/*
* udata[1-3] = unused
*/
if ((error = kcpc_restart(t->t_cpc_set)) != 0)
return (set_errno(error));
}
return (0);
case CPC_ENABLE:
case CPC_DISABLE:
udata1 = 0;
/*FALLTHROUGH*/
case CPC_USR_EVENTS:
case CPC_SYS_EVENTS:
if (t != curthread || t->t_cpc_set == NULL)
return (set_errno(EINVAL));
/*
* Provided for backwards compatibility with CPCv1.
*
* Stop the counters and record the current counts. Use the
* counts as the preset to rebind a new set with the requests
* reconfigured as requested.
*
* udata1: 1 == enable; 0 == disable
* udata{2,3}: unused
*/
rw_enter(&kcpc_cpuctx_lock, RW_READER);
if ((error = kcpc_enable(t,
cmd, (int)(uintptr_t)udata1)) != 0) {
rw_exit(&kcpc_cpuctx_lock);
return (set_errno(error));
}
rw_exit(&kcpc_cpuctx_lock);
return (0);
case CPC_NPIC:
return (cpc_ncounters);
case CPC_CAPS:
return (pcbe_ops->pcbe_caps);
case CPC_EVLIST_SIZE:
case CPC_LIST_EVENTS:
/*
* udata1 = pointer to user's int or buffer
* udata2 = picnum
* udata3 = unused
*/
if ((uintptr_t)udata2 >= cpc_ncounters)
return (set_errno(EINVAL));
size = strlen(
pcbe_ops->pcbe_list_events((uintptr_t)udata2)) + 1;
if (cmd == CPC_EVLIST_SIZE) {
if (suword32(udata1, size) == -1)
return (set_errno(EFAULT));
} else {
if (copyout(
pcbe_ops->pcbe_list_events((uintptr_t)udata2),
udata1, size) == -1)
return (set_errno(EFAULT));
}
return (0);
case CPC_ATTRLIST_SIZE:
case CPC_LIST_ATTRS:
/*
* udata1 = pointer to user's int or buffer
* udata2 = unused
* udata3 = unused
*
* attrlist size is length of PCBE-supported attributes, plus
* room for "picnum\0" plus an optional ',' separator char.
*/
str = pcbe_ops->pcbe_list_attrs();
size = strlen(str) + sizeof (SEPARATOR ATTRLIST) + 1;
if (str[0] != '\0')
/*
* A ',' separator character is necessary.
*/
size += 1;
if (cmd == CPC_ATTRLIST_SIZE) {
if (suword32(udata1, size) == -1)
return (set_errno(EFAULT));
} else {
/*
* Copyout the PCBE attributes, and then append the
* generic attribute list (with separator if necessary).
*/
if (copyout(str, udata1, strlen(str)) == -1)
return (set_errno(EFAULT));
if (str[0] != '\0') {
if (copyout(SEPARATOR ATTRLIST,
((char *)udata1) + strlen(str),
strlen(SEPARATOR ATTRLIST) + 1)
== -1)
return (set_errno(EFAULT));
} else
if (copyout(ATTRLIST,
(char *)udata1 + strlen(str),
strlen(ATTRLIST) + 1) == -1)
return (set_errno(EFAULT));
}
return (0);
case CPC_IMPL_NAME:
case CPC_CPUREF:
/*
* udata1 = pointer to user's buffer
* udata2 = unused
* udata3 = unused
*/
if (cmd == CPC_IMPL_NAME) {
str = pcbe_ops->pcbe_impl_name();
ASSERT(strlen(str) < CPC_MAX_IMPL_NAME);
} else {
str = pcbe_ops->pcbe_cpuref();
ASSERT(strlen(str) < CPC_MAX_CPUREF);
}
if (copyout(str, udata1, strlen(str) + 1) != 0)
return (set_errno(EFAULT));
return (0);
case CPC_INVALIDATE:
kcpc_invalidate(t);
return (0);
case CPC_RELE:
if ((error = kcpc_unbind(t->t_cpc_set)) != 0)
return (set_errno(error));
return (0);
default:
return (set_errno(EINVAL));
}
}
/*ARGSUSED*/
static int
kcpc_ioctl(dev_t dev, int cmd, intptr_t data, int flags, cred_t *cr, int *rvp)
{
kthread_t *t = curthread;
processorid_t cpuid;
void *udata1 = NULL;
void *udata2 = NULL;
void *udata3 = NULL;
int error;
int code;
STRUCT_DECL(__cpc_args, args);
STRUCT_INIT(args, flags);
if (curthread->t_bind_cpu != getminor(dev))
return (EAGAIN); /* someone unbound it? */
cpuid = getminor(dev);
if (cmd == CPCIO_BIND || cmd == CPCIO_SAMPLE) {
if (copyin((void *)data, STRUCT_BUF(args),
STRUCT_SIZE(args)) == -1)
return (EFAULT);
udata1 = STRUCT_FGETP(args, udata1);
udata2 = STRUCT_FGETP(args, udata2);
udata3 = STRUCT_FGETP(args, udata3);
}
switch (cmd) {
case CPCIO_BIND:
/*
* udata1 = pointer to packed nvlist buffer
* udata2 = size of packed nvlist buffer
* udata3 = User addr to return error subcode in.
*/
if (t->t_cpc_set != NULL) {
(void) kcpc_unbind(t->t_cpc_set);
ASSERT(t->t_cpc_set == NULL);
}
if ((error = kcpc_copyin_set(&t->t_cpc_set, udata1,
(size_t)udata2)) != 0) {
return (error);
}
if ((error = kcpc_verify_set(t->t_cpc_set)) != 0) {
kcpc_free_set(t->t_cpc_set);
t->t_cpc_set = NULL;
if (copyout(&error, udata3, sizeof (error)) == -1)
return (EFAULT);
return (EINVAL);
}
if ((error = kcpc_bind_cpu(t->t_cpc_set, cpuid, &code)) != 0) {
kcpc_free_set(t->t_cpc_set);
t->t_cpc_set = NULL;
/*
* Subcodes are only returned for EINVAL and EACCESS.
*/
if ((error == EINVAL || error == EACCES) &&
copyout(&code, udata3, sizeof (code)) == -1)
return (EFAULT);
return (error);
}
return (0);
case CPCIO_SAMPLE:
/*
* udata1 = pointer to user's buffer
* udata2 = pointer to user's hrtime
* udata3 = pointer to user's tick
*/
/*
* Only CPU-bound sets may be sampled via the ioctl(). If this
* set has no CPU-bound context, return an error.
*/
if (t->t_cpc_set == NULL)
return (EINVAL);
if ((error = kcpc_sample(t->t_cpc_set, udata1, udata2,
udata3)) != 0)
return (error);
return (0);
case CPCIO_RELE:
if (t->t_cpc_set == NULL)
return (EINVAL);
return (kcpc_unbind(t->t_cpc_set));
default:
return (EINVAL);
}
}
/*
* Caller must hold kcpc_cpuctx_lock.
*/
int
kcpc_enable(kthread_t *t, int cmd, int enable)
{
kcpc_ctx_t *ctx = t->t_cpc_ctx;
kcpc_set_t *set = t->t_cpc_set;
kcpc_set_t *newset;
int i;
int flag;
int err;
ASSERT(RW_READ_HELD(&kcpc_cpuctx_lock));
if (ctx == NULL) {
/*
* This thread has a set but no context; it must be a
* CPU-bound set.
*/
ASSERT(t->t_cpc_set != NULL);
ASSERT(t->t_cpc_set->ks_ctx->kc_cpuid != -1);
return (EINVAL);
} else if (ctx->kc_flags & KCPC_CTX_INVALID)
return (EAGAIN);
if (cmd == CPC_ENABLE) {
if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0)
return (EINVAL);
kpreempt_disable();
atomic_and_uint(&ctx->kc_flags, ~KCPC_CTX_FREEZE);
kcpc_restore(ctx);
kpreempt_enable();
} else if (cmd == CPC_DISABLE) {
if (ctx->kc_flags & KCPC_CTX_FREEZE)
return (EINVAL);
kpreempt_disable();
kcpc_save(ctx);
atomic_or_uint(&ctx->kc_flags, KCPC_CTX_FREEZE);
kpreempt_enable();
} else if (cmd == CPC_USR_EVENTS || cmd == CPC_SYS_EVENTS) {
/*
* Strategy for usr/sys: stop counters and update set's presets
* with current counter values, unbind, update requests with
* new config, then re-bind.
*/
flag = (cmd == CPC_USR_EVENTS) ?
CPC_COUNT_USER: CPC_COUNT_SYSTEM;
kpreempt_disable();
atomic_or_uint(&ctx->kc_flags,
KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);
pcbe_ops->pcbe_allstop();
kpreempt_enable();
for (i = 0; i < set->ks_nreqs; i++) {
set->ks_req[i].kr_preset = *(set->ks_req[i].kr_data);
if (enable)
set->ks_req[i].kr_flags |= flag;
else
set->ks_req[i].kr_flags &= ~flag;
}
newset = kcpc_dup_set(set);
if (kcpc_unbind(set) != 0)
return (EINVAL);
t->t_cpc_set = newset;
if (kcpc_bind_thread(newset, t, &err) != 0) {
t->t_cpc_set = NULL;
kcpc_free_set(newset);
return (EINVAL);
}
} else
return (EINVAL);
return (0);
}
