uint_t
get_syscall_args(klwp_t *lwp, long *argp, int *nargsp)
{
kthread_t *t = lwptot(lwp);
ulong_t mask = 0xfffffffful;
uint_t code;
long *ap;
int nargs;
#if defined(_LP64)
if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
mask = 0xfffffffffffffffful;
#endif
/*
* The thread lock must be held while looking at the arguments to ensure
* they don't go away via post_syscall().
* get_syscall_args() is the only routine to read them which is callable
* outside the LWP in question and hence the only one that must be
* synchronized in this manner.
*/
thread_lock(t);
code = t->t_sysnum;
ap = lwp->lwp_ap;
thread_unlock(t);
if (code != 0 && code < NSYSCALL) {
nargs = LWP_GETSYSENT(lwp)[code].sy_narg;
ASSERT(nargs <= MAXSYSARGS);
*nargsp = nargs;
while (nargs-- > 0)
*argp++ = *ap++ & mask;
} else {
*nargsp = 0;
}
return (code);
}
/*
* Perform pre-system-call processing, including stopping for tracing,
* auditing, etc.
*
* This routine is called only if the t_pre_sys flag is set. Any condition
* requiring pre-syscall handling must set the t_pre_sys flag. If the
* condition is persistent, this routine will repost t_pre_sys.
*/
int
pre_syscall()
{
kthread_t *t = curthread;
unsigned code = t->t_sysnum;
klwp_t *lwp = ttolwp(t);
proc_t *p = ttoproc(t);
int repost;
t->t_pre_sys = repost = 0; /* clear pre-syscall processing flag */
ASSERT(t->t_schedflag & TS_DONT_SWAP);
#if defined(DEBUG)
/*
* On the i386 kernel, lwp_ap points at the piece of the thread
* stack that we copy the users arguments into.
*
* On the amd64 kernel, the syscall arguments in the rdi..r9
* registers should be pointed at by lwp_ap. If the args need to
* be copied so that those registers can be changed without losing
* the ability to get the args for /proc, they can be saved by
* save_syscall_args(), and lwp_ap will be restored by post_syscall().
*/
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi);
} else {
#endif
ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase &&
(caddr_t)lwp->lwp_ap < t->t_stk);
}
#endif /* DEBUG */
/*
* Make sure the thread is holding the latest credentials for the
* process. The credentials in the process right now apply to this
* thread for the entire system call.
*/
if (t->t_cred != p->p_cred) {
cred_t *oldcred = t->t_cred;
/*
* DTrace accesses t_cred in probe context. t_cred must
* always be either NULL, or point to a valid, allocated cred
* structure.
*/
t->t_cred = crgetcred();
crfree(oldcred);
}
/*
* From the proc(4) manual page:
* When entry to a system call is being traced, the traced process
* stops after having begun the call to the system but before the
* system call arguments have been fetched from the process.
*/
if (PTOU(p)->u_systrap) {
if (prismember(&PTOU(p)->u_entrymask, code)) {
mutex_enter(&p->p_lock);
/*
* Recheck stop condition, now that lock is held.
*/
if (PTOU(p)->u_systrap &&
prismember(&PTOU(p)->u_entrymask, code)) {
stop(PR_SYSENTRY, code);
/*
* /proc may have modified syscall args,
* either in regs for amd64 or on ustack
* for ia32. Either way, arrange to
* copy them again, both for the syscall
* handler and for other consumers in
* post_syscall (like audit). Here, we
* only do amd64, and just set lwp_ap
* back to the kernel-entry stack copy;
* the syscall ml code redoes
* move-from-regs to set up for the
* syscall handler after we return. For
* ia32, save_syscall_args() below makes
* an lwp_ap-accessible copy.
*/
#if defined(_LP64)
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
lwp->lwp_argsaved = 0;
lwp->lwp_ap =
(long *)&lwptoregs(lwp)->r_rdi;
}
#endif
}
mutex_exit(&p->p_lock);
}
repost = 1;
}
/*
* ia32 kernel, or ia32 proc on amd64 kernel: keep args in
* lwp_arg for post-syscall processing, regardless of whether
* they might have been changed in /proc above.
*/
#if defined(_LP64)
if (lwp_getdatamodel(lwp) != DATAMODEL_NATIVE)
#endif
(void) save_syscall_args();
if (lwp->lwp_sysabort) {
/*
* lwp_sysabort may have been set via /proc while the process
* was stopped on PR_SYSENTRY. If so, abort the system call.
* Override any error from the copyin() of the arguments.
*/
lwp->lwp_sysabort = 0;
(void) set_errno(EINTR); /* forces post_sys */
t->t_pre_sys = 1; /* repost anyway */
return (1); /* don't do system call, return EINTR */
}
#ifdef C2_AUDIT
if (audit_active) { /* begin auditing for this syscall */
int error;
if (error = audit_start(T_SYSCALL, code, 0, lwp)) {
t->t_pre_sys = 1; /* repost anyway */
(void) set_errno(error);
return (1);
}
repost = 1;
}
#endif /* C2_AUDIT */
#ifndef NPROBE
/* Kernel probe */
if (tnf_tracing_active) {
TNF_PROBE_1(syscall_start, "syscall thread", /* CSTYLED */,
tnf_sysnum, sysnum, t->t_sysnum);
t->t_post_sys = 1; /* make sure post_syscall runs */
repost = 1;
}
#endif /* NPROBE */
#ifdef SYSCALLTRACE
if (syscalltrace) {
int i;
long *ap;
char *cp;
char *sysname;
struct sysent *callp;
if (code >= NSYSCALL)
callp = &nosys_ent; /* nosys has no args */
else
callp = LWP_GETSYSENT(lwp) + code;
(void) save_syscall_args();
mutex_enter(&systrace_lock);
printf("%d: ", p->p_pid);
if (code >= NSYSCALL)
printf("0x%x", code);
else {
sysname = mod_getsysname(code);
printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" :
sysname, code, callp->sy_callc);
}
cp = "(";
for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) {
printf("%s%lx", cp, *ap);
cp = ", ";
}
if (i)
printf(")");
printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread);
mutex_exit(&systrace_lock);
}
#endif /* SYSCALLTRACE */
/*
* If there was a continuing reason for pre-syscall processing,
* set the t_pre_sys flag for the next system call.
*/
if (repost)
t->t_pre_sys = 1;
lwp->lwp_error = 0; /* for old drivers */
lwp->lwp_badpriv = PRIV_NONE;
return (0);
}
/*
* Post-syscall processing. Perform abnormal system call completion
* actions such as /proc tracing, profiling, signals, preemption, etc.
*
* This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
* Any condition requiring pre-syscall handling must set one of these.
* If the condition is persistent, this routine will repost t_post_sys.
*/
void
post_syscall(long rval1, long rval2)
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
proc_t *p = ttoproc(t);
struct regs *rp = lwptoregs(lwp);
uint_t error;
uint_t code = t->t_sysnum;
int repost = 0;
int proc_stop = 0; /* non-zero if stopping */
int sigprof = 0; /* non-zero if sending SIGPROF */
t->t_post_sys = 0;
error = lwp->lwp_errno;
/*
* Code can be zero if this is a new LWP returning after a forkall(),
* other than the one which matches the one in the parent which called
* forkall(). In these LWPs, skip most of post-syscall activity.
*/
if (code == 0)
goto sig_check;
/*
* If the trace flag is set, mark the lwp to take a single-step trap
* on return to user level (below). The x86 lcall interface and
* sysenter has already done this, and turned off the flag, but
* amd64 syscall interface has not.
*/
if (rp->r_ps & PS_T) {
lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
rp->r_ps &= ~PS_T;
aston(curthread);
}
#ifdef C2_AUDIT
if (audit_active) { /* put out audit record for this syscall */
rval_t rval;
/* XX64 -- truncation of 64-bit return values? */
rval.r_val1 = (int)rval1;
rval.r_val2 = (int)rval2;
audit_finish(T_SYSCALL, code, error, &rval);
repost = 1;
}
#endif /* C2_AUDIT */
if (curthread->t_pdmsg != NULL) {
char *m = curthread->t_pdmsg;
uprintf("%s", m);
kmem_free(m, strlen(m) + 1);
curthread->t_pdmsg = NULL;
}
/*
* If we're going to stop for /proc tracing, set the flag and
* save the arguments so that the return values don't smash them.
*/
if (PTOU(p)->u_systrap) {
if (prismember(&PTOU(p)->u_exitmask, code)) {
if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
(void) save_syscall_args();
proc_stop = 1;
}
repost = 1;
}
/*
* Similarly check to see if SIGPROF might be sent.
*/
if (curthread->t_rprof != NULL &&
curthread->t_rprof->rp_anystate != 0) {
if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
(void) save_syscall_args();
sigprof = 1;
}
if (lwp->lwp_eosys == NORMALRETURN) {
if (error == 0) {
#ifdef SYSCALLTRACE
if (syscalltrace) {
mutex_enter(&systrace_lock);
printf(
"%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
p->p_pid, rval1, rval2, curthread);
mutex_exit(&systrace_lock);
}
#endif /* SYSCALLTRACE */
rp->r_ps &= ~PS_C;
rp->r_r0 = rval1;
rp->r_r1 = rval2;
} else {
int sig;
#ifdef SYSCALLTRACE
if (syscalltrace) {
mutex_enter(&systrace_lock);
printf("%d: error=%d, id 0x%p\n",
p->p_pid, error, curthread);
mutex_exit(&systrace_lock);
}
#endif /* SYSCALLTRACE */
if (error == EINTR && t->t_activefd.a_stale)
error = EBADF;
if (error == EINTR &&
(sig = lwp->lwp_cursig) != 0 &&
sigismember(&PTOU(p)->u_sigrestart, sig) &&
PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
PTOU(p)->u_signal[sig - 1] != SIG_IGN)
error = ERESTART;
rp->r_r0 = error;
rp->r_ps |= PS_C;
}
}
/*
* From the proc(4) manual page:
* When exit from a system call is being traced, the traced process
* stops on completion of the system call just prior to checking for
* signals and returning to user level. At this point all return
* values have been stored into the traced process's saved registers.
*/
if (proc_stop) {
mutex_enter(&p->p_lock);
if (PTOU(p)->u_systrap &&
prismember(&PTOU(p)->u_exitmask, code))
stop(PR_SYSEXIT, code);
mutex_exit(&p->p_lock);
}
/*
* If we are the parent returning from a successful
* vfork, wait for the child to exec or exit.
* This code must be here and not in the bowels of the system
* so that /proc can intercept exit from vfork in a timely way.
*/
if (code == SYS_vfork && rp->r_r1 == 0 && error == 0)
vfwait((pid_t)rval1);
/*
* If profiling is active, bill the current PC in user-land
* and keep reposting until profiling is disabled.
*/
if (p->p_prof.pr_scale) {
if (lwp->lwp_oweupc)
profil_tick(rp->r_pc);
repost = 1;
}
sig_check:
/*
* Reset flag for next time.
* We must do this after stopping on PR_SYSEXIT
* because /proc uses the information in lwp_eosys.
*/
lwp->lwp_eosys = NORMALRETURN;
clear_stale_fd();
t->t_flag &= ~T_FORKALL;
if (t->t_astflag | t->t_sig_check) {
/*
* Turn off the AST flag before checking all the conditions that
* may have caused an AST. This flag is on whenever a signal or
* unusual condition should be handled after the next trap or
* syscall.
*/
astoff(t);
/*
* If a single-step trap occurred on a syscall (see trap())
* recognize it now. Do this before checking for signals
* because deferred_singlestep_trap() may generate a SIGTRAP to
* the LWP or may otherwise mark the LWP to call issig(FORREAL).
*/
if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
deferred_singlestep_trap((caddr_t)rp->r_pc);
t->t_sig_check = 0;
/*
* The following check is legal for the following reasons:
* 1) The thread we are checking, is ourselves, so there is
* no way the proc can go away.
* 2) The only time we need to be protected by the
* lock is if the binding is changed.
*
* Note we will still take the lock and check the binding
* if the condition was true without the lock held. This
* prevents lock contention among threads owned by the
* same proc.
*/
if (curthread->t_proc_flag & TP_CHANGEBIND) {
mutex_enter(&p->p_lock);
if (curthread->t_proc_flag & TP_CHANGEBIND) {
timer_lwpbind();
curthread->t_proc_flag &= ~TP_CHANGEBIND;
}
mutex_exit(&p->p_lock);
}
/*
* for kaio requests on the special kaio poll queue,
* copyout their results to user memory.
*/
if (p->p_aio)
aio_cleanup(0);
/*
* If this LWP was asked to hold, call holdlwp(), which will
* stop. holdlwps() sets this up and calls pokelwps() which
* sets the AST flag.
*
* Also check TP_EXITLWP, since this is used by fresh new LWPs
* through lwp_rtt(). That flag is set if the lwp_create(2)
* syscall failed after creating the LWP.
*/
if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
holdlwp();
/*
* All code that sets signals and makes ISSIG_PENDING
* evaluate true must set t_sig_check afterwards.
*/
if (ISSIG_PENDING(t, lwp, p)) {
if (issig(FORREAL))
psig();
t->t_sig_check = 1; /* recheck next time */
}
if (sigprof) {
realsigprof(code, error);
t->t_sig_check = 1; /* recheck next time */
}
/*
* If a performance counter overflow interrupt was
* delivered *during* the syscall, then re-enable the
* AST so that we take a trip through trap() to cause
* the SIGEMT to be delivered.
*/
if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
aston(t);
/*
* /proc can't enable/disable the trace bit itself
* because that could race with the call gate used by
* system calls via "lcall". If that happened, an
* invalid EFLAGS would result. prstep()/prnostep()
* therefore schedule an AST for the purpose.
*/
if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
rp->r_ps |= PS_T;
}
if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
rp->r_ps &= ~PS_T;
}
}
lwp->lwp_errno = 0; /* clear error for next time */
#ifndef NPROBE
/* Kernel probe */
if (tnf_tracing_active) {
TNF_PROBE_3(syscall_end, "syscall thread", /* CSTYLED */,
tnf_long, rval1, rval1,
tnf_long, rval2, rval2,
tnf_long, errno, (long)error);
repost = 1;
}
#endif /* NPROBE */
/*
* Set state to LWP_USER here so preempt won't give us a kernel
* priority if it occurs after this point. Call CL_TRAPRET() to
* restore the user-level priority.
*
* It is important that no locks (other than spinlocks) be entered
* after this point before returning to user mode (unless lwp_state
* is set back to LWP_SYS).
*
* XXX Sampled times past this point are charged to the user.
*/
lwp->lwp_state = LWP_USER;
if (t->t_trapret) {
t->t_trapret = 0;
thread_lock(t);
CL_TRAPRET(t);
thread_unlock(t);
}
if (CPU->cpu_runrun)
preempt();
lwp->lwp_errno = 0; /* clear error for next time */
/*
* The thread lock must be held in order to clear sysnum and reset
* lwp_ap atomically with respect to other threads in the system that
* may be looking at the args via lwp_ap from get_syscall_args().
*/
thread_lock(t);
t->t_sysnum = 0; /* no longer in a system call */
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
/*
* In case the args were copied to the lwp, reset the
* pointer so the next syscall will have the right
* lwp_ap pointer.
*/
lwp->lwp_ap = (long *)&rp->r_rdi;
} else {
#endif
lwp->lwp_ap = NULL; /* reset on every syscall entry */
}
thread_unlock(t);
lwp->lwp_argsaved = 0;
/*
* If there was a continuing reason for post-syscall processing,
* set the t_post_sys flag for the next system call.
*/
if (repost)
t->t_post_sys = 1;
/*
* If there is a ustack registered for this lwp, and the stack rlimit
* has been altered, read in the ustack. If the saved stack rlimit
* matches the bounds of the ustack, update the ustack to reflect
* the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
* stack checking by setting the size to 0.
*/
if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
rlim64_t new_size;
caddr_t top;
stack_t stk;
struct rlimit64 rl;
mutex_enter(&p->p_lock);
new_size = p->p_stk_ctl;
top = p->p_usrstack;
(void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
mutex_exit(&p->p_lock);
if (rl.rlim_cur == RLIM64_INFINITY)
new_size = 0;
if (copyin((stack_t *)lwp->lwp_ustack, &stk,
sizeof (stack_t)) == 0 &&
(stk.ss_size == lwp->lwp_old_stk_ctl ||
stk.ss_size == 0) &&
stk.ss_sp == top - stk.ss_size) {
stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
stk.ss_size - (uintptr_t)new_size);
stk.ss_size = new_size;
(void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
sizeof (stack_t));
}
lwp->lwp_old_stk_ctl = 0;
}
}
/*
* Save the system call arguments in a safe place.
*
* On the i386 kernel:
*
* Copy the users args prior to changing the stack or stack pointer.
* This is so /proc will be able to get a valid copy of the
* args from the user stack even after the user stack has been changed.
* Note that the kernel stack copy of the args may also have been
* changed by a system call handler which takes C-style arguments.
*
* Note that this may be called by stop() from trap(). In that case
* t_sysnum will be zero (syscall_exit clears it), so no args will be
* copied.
*
* On the amd64 kernel:
*
* For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9
* in the reg structure. If the user is going to change the argument
* registers, rax, or the stack and might want to get the args (for
* /proc tracing), it must copy the args elsewhere via save_syscall_args().
*
* For 32-bit applications, lwp->lwp_ap normally points to a copy of
* the system call arguments on the kernel stack made from the user
* stack. Copy the args prior to change the stack or stack pointer.
* This is so /proc will be able to get a valid copy of the args
* from the user stack even after that stack has been changed.
*
* This may be called from stop() even when we're not in a system call.
* Since there's no easy way to tell, this must be safe (not panic).
* If the copyins get data faults, return non-zero.
*/
int
save_syscall_args()
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
uint_t code = t->t_sysnum;
uint_t nargs;
if (lwp->lwp_argsaved || code == 0)
return (0); /* args already saved or not needed */
if (code >= NSYSCALL) {
nargs = 0; /* illegal syscall */
} else {
struct sysent *se = LWP_GETSYSENT(lwp);
struct sysent *callp = se + code;
nargs = callp->sy_narg;
if (LOADABLE_SYSCALL(callp) && nargs == 0) {
krwlock_t *module_lock;
/*
* Find out how many arguments the system
* call uses.
*
* We have the property that loaded syscalls
* never change the number of arguments they
* use after they've been loaded once. This
* allows us to stop for /proc tracing without
* holding the module lock.
* /proc is assured that sy_narg is valid.
*/
module_lock = lock_syscall(se, code);
nargs = callp->sy_narg;
rw_exit(module_lock);
}
}
/*
* Fetch the system call arguments.
*/
if (nargs == 0)
goto out;
ASSERT(nargs <= MAXSYSARGS);
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
struct regs *rp = lwptoregs(lwp);
lwp->lwp_arg[0] = rp->r_rdi;
lwp->lwp_arg[1] = rp->r_rsi;
lwp->lwp_arg[2] = rp->r_rdx;
lwp->lwp_arg[3] = rp->r_rcx;
lwp->lwp_arg[4] = rp->r_r8;
lwp->lwp_arg[5] = rp->r_r9;
if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6))
return (-1);
} else {
#endif
if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs))
return (-1);
}
out:
lwp->lwp_ap = lwp->lwp_arg;
lwp->lwp_argsaved = 1;
t->t_post_sys = 1; /* so lwp_ap will be reset */
return (0);
}
/*
* Save the system call arguments in a safe place.
* lwp->lwp_ap normally points to the out regs in the reg structure.
* If the user is going to change the out registers, g1, or the stack,
* and might want to get the args (for /proc tracing), it must copy
* the args elsewhere via save_syscall_args().
*
* This may be called from stop() even when we're not in a system call.
* Since there's no easy way to tell, this must be safe (not panic).
* If the copyins get data faults, return non-zero.
*/
int
save_syscall_args()
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
struct regs *rp = lwptoregs(lwp);
uint_t code = t->t_sysnum;
uint_t nargs;
int i;
caddr_t ua;
model_t datamodel;
if (lwp->lwp_argsaved || code == 0)
return (0); /* args already saved or not needed */
if (code >= NSYSCALL) {
nargs = 0; /* illegal syscall */
} else {
struct sysent *se = LWP_GETSYSENT(lwp);
struct sysent *callp = se + code;
nargs = callp->sy_narg;
if (LOADABLE_SYSCALL(callp) && nargs == 0) {
krwlock_t *module_lock;
/*
* Find out how many arguments the system
* call uses.
*
* We have the property that loaded syscalls
* never change the number of arguments they
* use after they've been loaded once. This
* allows us to stop for /proc tracing without
* holding the module lock.
* /proc is assured that sy_narg is valid.
*/
module_lock = lock_syscall(se, code);
nargs = callp->sy_narg;
rw_exit(module_lock);
}
}
/*
* Fetch the system call arguments.
*/
if (nargs == 0)
goto out;
ASSERT(nargs <= MAXSYSARGS);
if ((datamodel = lwp_getdatamodel(lwp)) == DATAMODEL_ILP32) {
if (rp->r_g1 == 0) { /* indirect syscall */
lwp->lwp_arg[0] = (uint32_t)rp->r_o1;
lwp->lwp_arg[1] = (uint32_t)rp->r_o2;
lwp->lwp_arg[2] = (uint32_t)rp->r_o3;
lwp->lwp_arg[3] = (uint32_t)rp->r_o4;
lwp->lwp_arg[4] = (uint32_t)rp->r_o5;
if (nargs > 5) {
ua = (caddr_t)(uintptr_t)(caddr32_t)(uintptr_t)
(rp->r_sp + MINFRAME32);
for (i = 5; i < nargs; i++) {
uint32_t a;
if (fuword32(ua, &a) != 0)
return (-1);
lwp->lwp_arg[i] = a;
ua += sizeof (a);
}
}
} else {
lwp->lwp_arg[0] = (uint32_t)rp->r_o0;
lwp->lwp_arg[1] = (uint32_t)rp->r_o1;
lwp->lwp_arg[2] = (uint32_t)rp->r_o2;
lwp->lwp_arg[3] = (uint32_t)rp->r_o3;
lwp->lwp_arg[4] = (uint32_t)rp->r_o4;
lwp->lwp_arg[5] = (uint32_t)rp->r_o5;
if (nargs > 6) {
ua = (caddr_t)(uintptr_t)(caddr32_t)(uintptr_t)
(rp->r_sp + MINFRAME32);
for (i = 6; i < nargs; i++) {
uint32_t a;
if (fuword32(ua, &a) != 0)
return (-1);
lwp->lwp_arg[i] = a;
ua += sizeof (a);
}
}
}
} else {
ASSERT(datamodel == DATAMODEL_LP64);
lwp->lwp_arg[0] = rp->r_o0;
lwp->lwp_arg[1] = rp->r_o1;
lwp->lwp_arg[2] = rp->r_o2;
lwp->lwp_arg[3] = rp->r_o3;
lwp->lwp_arg[4] = rp->r_o4;
lwp->lwp_arg[5] = rp->r_o5;
if (nargs > 6) {
ua = (caddr_t)rp->r_sp + MINFRAME + STACK_BIAS;
for (i = 6; i < nargs; i++) {
unsigned long a;
if (fulword(ua, &a) != 0)
return (-1);
lwp->lwp_arg[i] = a;
ua += sizeof (a);
}
}
}
out:
lwp->lwp_ap = lwp->lwp_arg;
lwp->lwp_argsaved = 1;
t->t_post_sys = 1; /* so lwp_ap will be reset */
return (0);
}
int
lwp_setprivate(klwp_t *lwp, int which, uintptr_t base)
{
pcb_t *pcb = &lwp->lwp_pcb;
struct regs *rp = lwptoregs(lwp);
kthread_t *t = lwptot(lwp);
int thisthread = t == curthread;
int rval;
if (thisthread)
kpreempt_disable();
#if defined(__amd64)
/*
* 32-bit compatibility processes point to the per-cpu GDT segment
* descriptors that are virtualized to the lwp. That allows 32-bit
* programs to mess with %fs and %gs; in particular it allows
* things like this:
*
* movw %gs, %ax
* ...
* movw %ax, %gs
*
* to work, which is needed by emulators for legacy application
* environments ..
*
* 64-bit processes may also point to a per-cpu GDT segment descriptor
* virtualized to the lwp. However the descriptor base is forced
* to zero (because we can't express the full 64-bit address range
* in a long mode descriptor), so don't reload segment registers
* in a 64-bit program! 64-bit processes must have selector values
* of zero for %fs and %gs to use the 64-bit fs_base and gs_base
* respectively.
*/
if (pcb->pcb_rupdate == 0) {
pcb->pcb_ds = rp->r_ds;
pcb->pcb_es = rp->r_es;
pcb->pcb_fs = rp->r_fs;
pcb->pcb_gs = rp->r_gs;
pcb->pcb_rupdate = 1;
t->t_post_sys = 1;
}
ASSERT(t->t_post_sys);
switch (which) {
case _LWP_FSBASE:
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
set_usegd(&pcb->pcb_fsdesc, SDP_LONG, 0, 0,
SDT_MEMRWA, SEL_UPL, SDP_BYTES, SDP_OP32);
rval = pcb->pcb_fs = 0; /* null gdt descriptor */
} else {
set_usegd(&pcb->pcb_fsdesc, SDP_SHORT, (void *)base, -1,
SDT_MEMRWA, SEL_UPL, SDP_PAGES, SDP_OP32);
rval = pcb->pcb_fs = LWPFS_SEL;
}
if (thisthread)
gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
pcb->pcb_fsbase = base;
break;
case _LWP_GSBASE:
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
set_usegd(&pcb->pcb_gsdesc, SDP_LONG, 0, 0,
SDT_MEMRWA, SEL_UPL, SDP_BYTES, SDP_OP32);
rval = pcb->pcb_gs = 0; /* null gdt descriptor */
} else {
set_usegd(&pcb->pcb_gsdesc, SDP_SHORT, (void *)base, -1,
SDT_MEMRWA, SEL_UPL, SDP_PAGES, SDP_OP32);
rval = pcb->pcb_gs = LWPGS_SEL;
}
if (thisthread)
gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
pcb->pcb_gsbase = base;
break;
default:
rval = -1;
break;
}
#elif defined(__i386)
/*
* 32-bit processes point to the per-cpu GDT segment
* descriptors that are virtualized to the lwp.
*/
switch (which) {
case _LWP_FSBASE:
set_usegd(&pcb->pcb_fsdesc, (void *)base, -1,
SDT_MEMRWA, SEL_UPL, SDP_PAGES, SDP_OP32);
if (thisthread)
gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
rval = rp->r_fs = LWPFS_SEL;
break;
case _LWP_GSBASE:
set_usegd(&pcb->pcb_gsdesc, (void *)base, -1,
SDT_MEMRWA, SEL_UPL, SDP_PAGES, SDP_OP32);
if (thisthread)
gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
rval = rp->r_gs = LWPGS_SEL;
break;
default:
rval = -1;
break;
}
#endif /* __i386 */
if (thisthread)
kpreempt_enable();
return (rval);
}
static int
lwp_getprivate(klwp_t *lwp, int which, uintptr_t base)
{
pcb_t *pcb = &lwp->lwp_pcb;
struct regs *rp = lwptoregs(lwp);
uintptr_t sbase;
int error = 0;
ASSERT(lwptot(lwp) == curthread);
kpreempt_disable();
switch (which) {
#if defined(__amd64)
case _LWP_FSBASE:
if ((sbase = pcb->pcb_fsbase) != 0) {
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
if (pcb->pcb_rupdate == 1) {
if (pcb->pcb_fs == 0)
break;
} else {
if (rp->r_fs == 0)
break;
}
} else {
if (pcb->pcb_rupdate == 1) {
if (pcb->pcb_fs == LWPFS_SEL)
break;
} else {
if (rp->r_fs == LWPFS_SEL)
break;
}
}
}
error = EINVAL;
break;
case _LWP_GSBASE:
if ((sbase = pcb->pcb_gsbase) != 0) {
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
if (pcb->pcb_rupdate == 1) {
if (pcb->pcb_gs == 0)
break;
} else {
if (rp->r_gs == 0)
break;
}
} else {
if (pcb->pcb_rupdate == 1) {
if (pcb->pcb_gs == LWPGS_SEL)
break;
} else {
if (rp->r_gs == LWPGS_SEL)
break;
}
}
}
error = EINVAL;
break;
#elif defined(__i386)
case _LWP_FSBASE:
if (rp->r_fs == LWPFS_SEL) {
sbase = USEGD_GETBASE(&pcb->pcb_fsdesc);
break;
}
error = EINVAL;
break;
case _LWP_GSBASE:
if (rp->r_gs == LWPGS_SEL) {
sbase = USEGD_GETBASE(&pcb->pcb_gsdesc);
break;
}
error = EINVAL;
break;
#endif /* __i386 */
default:
error = ENOTSUP;
break;
}
kpreempt_enable();
if (error != 0)
return (error);
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
if (sulword((void *)base, sbase) == -1)
error = EFAULT;
#if defined(_SYSCALL32_IMPL)
} else {
if (suword32((void *)base, (uint32_t)sbase) == -1)
error = EFAULT;
#endif
}
return (error);
}
/*
* Add any lwp-associated context handlers to the lwp at the beginning
* of the lwp's useful life.
*
* All paths which create lwp's invoke lwp_create(); lwp_create()
* invokes lwp_stk_init() which initializes the stack, sets up
* lwp_regs, and invokes this routine.
*
* All paths which destroy lwp's invoke lwp_exit() to rip the lwp
* apart and put it on 'lwp_deathrow'; if the lwp is destroyed it
* ends up in thread_free() which invokes freectx(t, 0) before
* invoking lwp_stk_fini(). When the lwp is recycled from death
* row, lwp_stk_fini() is invoked, then thread_free(), and thus
* freectx(t, 0) as before.
*
* In the case of exec, the surviving lwp is thoroughly scrubbed
* clean; exec invokes freectx(t, 1) to destroy associated contexts.
* On the way back to the new image, it invokes setregs() which
* in turn invokes this routine.
*/
void
lwp_installctx(klwp_t *lwp)
{
kthread_t *t = lwptot(lwp);
int thisthread = t == curthread;
#ifdef _SYSCALL32_IMPL
void (*restop)(klwp_t *) = lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ?
lwp_segregs_restore : lwp_segregs_restore32;
#else
void (*restop)(klwp_t *) = lwp_segregs_restore;
#endif
/*
* Install the basic lwp context handlers on each lwp.
*
* On the amd64 kernel, the context handlers are responsible for
* virtualizing %ds, %es, %fs, and %gs to the lwp. The register
* values are only ever changed via sys_rtt when the
* pcb->pcb_rupdate == 1. Only sys_rtt gets to clear the bit.
*
* On the i386 kernel, the context handlers are responsible for
* virtualizing %gs/%fs to the lwp by updating the per-cpu GDTs
*/
ASSERT(removectx(t, lwp, lwp_segregs_save, restop,
NULL, NULL, NULL, NULL) == 0);
if (thisthread)
kpreempt_disable();
installctx(t, lwp, lwp_segregs_save, restop,
NULL, NULL, NULL, NULL);
if (thisthread) {
/*
* Since we're the right thread, set the values in the GDT
*/
restop(lwp);
kpreempt_enable();
}
/*
* If we have sysenter/sysexit instructions enabled, we need
* to ensure that the hardware mechanism is kept up-to-date with the
* lwp's kernel stack pointer across context switches.
*
* sep_save zeros the sysenter stack pointer msr; sep_restore sets
* it to the lwp's kernel stack pointer (kstktop).
*/
if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
#if defined(__amd64)
caddr_t kstktop = (caddr_t)lwp->lwp_regs;
#elif defined(__i386)
caddr_t kstktop = ((caddr_t)lwp->lwp_regs - MINFRAME) +
SA(sizeof (struct regs) + MINFRAME);
#endif
ASSERT(removectx(t, kstktop,
sep_save, sep_restore, NULL, NULL, NULL, NULL) == 0);
if (thisthread)
kpreempt_disable();
installctx(t, kstktop,
sep_save, sep_restore, NULL, NULL, NULL, NULL);
if (thisthread) {
/*
* We're the right thread, so set the stack pointer
* for the first sysenter instruction to use
*/
sep_restore(kstktop);
kpreempt_enable();
}
}
if (PROC_IS_BRANDED(ttoproc(t)))
lwp_attach_brand_hdlrs(lwp);
}
