/*
* Define the code needed before returning to user mode, for
* trap, mem_access_fault, and syscall.
*/
static inline void
userret(struct proc *p, int pc, u_quad_t oticks)
{
int sig;
/* take pending signals */
while ((sig = CURSIG(p)) != 0)
psig(sig);
p->p_pri = p->p_usrpri;
if (want_ast) {
want_ast = 0;
if (p->p_flag & SOWEUPC) {
p->p_flag &= ~SOWEUPC;
ADDUPROF(p);
}
}
if (want_resched) {
/*
* Since we are curproc, a clock interrupt could
* change our priority without changing run queues
* (the running process is not kept on a run queue).
* If this happened after we setrq ourselves but
* before we swtch()'ed, we might not be on the queue
* indicated by our priority.
*/
(void) splstatclock();
setrq(p);
p->p_stats->p_ru.ru_nivcsw++;
swtch();
(void) spl0();
while ((sig = CURSIG(p)) != 0)
psig(sig);
}
/*
* If profiling, charge recent system time to the trapped pc.
*/
if (p->p_flag & SPROFIL)
addupc_task(p, pc, (int)(p->p_sticks - oticks));
curpri = p->p_pri;
}
/*
* Arrange for the real time profiling signal to be dispatched.
*/
void
realsigprof(int sysnum, int error)
{
proc_t *p;
klwp_t *lwp;
if (curthread->t_rprof->rp_anystate == 0)
return;
p = ttoproc(curthread);
lwp = ttolwp(curthread);
mutex_enter(&p->p_lock);
if (sigismember(&p->p_ignore, SIGPROF) ||
signal_is_blocked(curthread, SIGPROF)) {
mutex_exit(&p->p_lock);
return;
}
lwp->lwp_siginfo.si_signo = SIGPROF;
lwp->lwp_siginfo.si_code = PROF_SIG;
lwp->lwp_siginfo.si_errno = error;
hrt2ts(gethrtime(), &lwp->lwp_siginfo.si_tstamp);
lwp->lwp_siginfo.si_syscall = sysnum;
lwp->lwp_siginfo.si_nsysarg = (sysnum > 0 && sysnum < NSYSCALL) ?
LWP_GETSYSENT(lwp)[sysnum].sy_narg : 0;
lwp->lwp_siginfo.si_fault = lwp->lwp_lastfault;
lwp->lwp_siginfo.si_faddr = lwp->lwp_lastfaddr;
lwp->lwp_lastfault = 0;
lwp->lwp_lastfaddr = NULL;
sigtoproc(p, curthread, SIGPROF);
mutex_exit(&p->p_lock);
ASSERT(lwp->lwp_cursig == 0);
if (issig(FORREAL)) {
psig();
}
mutex_enter(&p->p_lock);
lwp->lwp_siginfo.si_signo = 0;
bzero(curthread->t_rprof, sizeof (*curthread->t_rprof));
mutex_exit(&p->p_lock);
}
/*
* Post-syscall processing. Perform abnormal system call completion
* actions such as /proc tracing, profiling, signals, preemption, etc.
*
* This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
* Any condition requiring pre-syscall handling must set one of these.
* If the condition is persistent, this routine will repost t_post_sys.
*/
void
post_syscall(long rval1, long rval2)
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
proc_t *p = ttoproc(t);
struct regs *rp = lwptoregs(lwp);
uint_t error;
uint_t code = t->t_sysnum;
int repost = 0;
int proc_stop = 0; /* non-zero if stopping */
int sigprof = 0; /* non-zero if sending SIGPROF */
t->t_post_sys = 0;
error = lwp->lwp_errno;
/*
* Code can be zero if this is a new LWP returning after a forkall(),
* other than the one which matches the one in the parent which called
* forkall(). In these LWPs, skip most of post-syscall activity.
*/
if (code == 0)
goto sig_check;
/*
* If the trace flag is set, mark the lwp to take a single-step trap
* on return to user level (below). The x86 lcall interface and
* sysenter has already done this, and turned off the flag, but
* amd64 syscall interface has not.
*/
if (rp->r_ps & PS_T) {
lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
rp->r_ps &= ~PS_T;
aston(curthread);
}
#ifdef C2_AUDIT
if (audit_active) { /* put out audit record for this syscall */
rval_t rval;
/* XX64 -- truncation of 64-bit return values? */
rval.r_val1 = (int)rval1;
rval.r_val2 = (int)rval2;
audit_finish(T_SYSCALL, code, error, &rval);
repost = 1;
}
#endif /* C2_AUDIT */
if (curthread->t_pdmsg != NULL) {
char *m = curthread->t_pdmsg;
uprintf("%s", m);
kmem_free(m, strlen(m) + 1);
curthread->t_pdmsg = NULL;
}
/*
* If we're going to stop for /proc tracing, set the flag and
* save the arguments so that the return values don't smash them.
*/
if (PTOU(p)->u_systrap) {
if (prismember(&PTOU(p)->u_exitmask, code)) {
if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
(void) save_syscall_args();
proc_stop = 1;
}
repost = 1;
}
/*
* Similarly check to see if SIGPROF might be sent.
*/
if (curthread->t_rprof != NULL &&
curthread->t_rprof->rp_anystate != 0) {
if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
(void) save_syscall_args();
sigprof = 1;
}
if (lwp->lwp_eosys == NORMALRETURN) {
if (error == 0) {
#ifdef SYSCALLTRACE
if (syscalltrace) {
mutex_enter(&systrace_lock);
printf(
"%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
p->p_pid, rval1, rval2, curthread);
mutex_exit(&systrace_lock);
}
#endif /* SYSCALLTRACE */
rp->r_ps &= ~PS_C;
rp->r_r0 = rval1;
rp->r_r1 = rval2;
} else {
int sig;
#ifdef SYSCALLTRACE
if (syscalltrace) {
mutex_enter(&systrace_lock);
printf("%d: error=%d, id 0x%p\n",
p->p_pid, error, curthread);
mutex_exit(&systrace_lock);
}
#endif /* SYSCALLTRACE */
if (error == EINTR && t->t_activefd.a_stale)
error = EBADF;
if (error == EINTR &&
(sig = lwp->lwp_cursig) != 0 &&
sigismember(&PTOU(p)->u_sigrestart, sig) &&
PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
PTOU(p)->u_signal[sig - 1] != SIG_IGN)
error = ERESTART;
rp->r_r0 = error;
rp->r_ps |= PS_C;
}
}
/*
* From the proc(4) manual page:
* When exit from a system call is being traced, the traced process
* stops on completion of the system call just prior to checking for
* signals and returning to user level. At this point all return
* values have been stored into the traced process's saved registers.
*/
if (proc_stop) {
mutex_enter(&p->p_lock);
if (PTOU(p)->u_systrap &&
prismember(&PTOU(p)->u_exitmask, code))
stop(PR_SYSEXIT, code);
mutex_exit(&p->p_lock);
}
/*
* If we are the parent returning from a successful
* vfork, wait for the child to exec or exit.
* This code must be here and not in the bowels of the system
* so that /proc can intercept exit from vfork in a timely way.
*/
if (code == SYS_vfork && rp->r_r1 == 0 && error == 0)
vfwait((pid_t)rval1);
/*
* If profiling is active, bill the current PC in user-land
* and keep reposting until profiling is disabled.
*/
if (p->p_prof.pr_scale) {
if (lwp->lwp_oweupc)
profil_tick(rp->r_pc);
repost = 1;
}
sig_check:
/*
* Reset flag for next time.
* We must do this after stopping on PR_SYSEXIT
* because /proc uses the information in lwp_eosys.
*/
lwp->lwp_eosys = NORMALRETURN;
clear_stale_fd();
t->t_flag &= ~T_FORKALL;
if (t->t_astflag | t->t_sig_check) {
/*
* Turn off the AST flag before checking all the conditions that
* may have caused an AST. This flag is on whenever a signal or
* unusual condition should be handled after the next trap or
* syscall.
*/
astoff(t);
/*
* If a single-step trap occurred on a syscall (see trap())
* recognize it now. Do this before checking for signals
* because deferred_singlestep_trap() may generate a SIGTRAP to
* the LWP or may otherwise mark the LWP to call issig(FORREAL).
*/
if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
deferred_singlestep_trap((caddr_t)rp->r_pc);
t->t_sig_check = 0;
/*
* The following check is legal for the following reasons:
* 1) The thread we are checking, is ourselves, so there is
* no way the proc can go away.
* 2) The only time we need to be protected by the
* lock is if the binding is changed.
*
* Note we will still take the lock and check the binding
* if the condition was true without the lock held. This
* prevents lock contention among threads owned by the
* same proc.
*/
if (curthread->t_proc_flag & TP_CHANGEBIND) {
mutex_enter(&p->p_lock);
if (curthread->t_proc_flag & TP_CHANGEBIND) {
timer_lwpbind();
curthread->t_proc_flag &= ~TP_CHANGEBIND;
}
mutex_exit(&p->p_lock);
}
/*
* for kaio requests on the special kaio poll queue,
* copyout their results to user memory.
*/
if (p->p_aio)
aio_cleanup(0);
/*
* If this LWP was asked to hold, call holdlwp(), which will
* stop. holdlwps() sets this up and calls pokelwps() which
* sets the AST flag.
*
* Also check TP_EXITLWP, since this is used by fresh new LWPs
* through lwp_rtt(). That flag is set if the lwp_create(2)
* syscall failed after creating the LWP.
*/
if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
holdlwp();
/*
* All code that sets signals and makes ISSIG_PENDING
* evaluate true must set t_sig_check afterwards.
*/
if (ISSIG_PENDING(t, lwp, p)) {
if (issig(FORREAL))
psig();
t->t_sig_check = 1; /* recheck next time */
}
if (sigprof) {
realsigprof(code, error);
t->t_sig_check = 1; /* recheck next time */
}
/*
* If a performance counter overflow interrupt was
* delivered *during* the syscall, then re-enable the
* AST so that we take a trip through trap() to cause
* the SIGEMT to be delivered.
*/
if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
aston(t);
/*
* /proc can't enable/disable the trace bit itself
* because that could race with the call gate used by
* system calls via "lcall". If that happened, an
* invalid EFLAGS would result. prstep()/prnostep()
* therefore schedule an AST for the purpose.
*/
if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
rp->r_ps |= PS_T;
}
if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
rp->r_ps &= ~PS_T;
}
}
lwp->lwp_errno = 0; /* clear error for next time */
#ifndef NPROBE
/* Kernel probe */
if (tnf_tracing_active) {
TNF_PROBE_3(syscall_end, "syscall thread", /* CSTYLED */,
tnf_long, rval1, rval1,
tnf_long, rval2, rval2,
tnf_long, errno, (long)error);
repost = 1;
}
#endif /* NPROBE */
/*
* Set state to LWP_USER here so preempt won't give us a kernel
* priority if it occurs after this point. Call CL_TRAPRET() to
* restore the user-level priority.
*
* It is important that no locks (other than spinlocks) be entered
* after this point before returning to user mode (unless lwp_state
* is set back to LWP_SYS).
*
* XXX Sampled times past this point are charged to the user.
*/
lwp->lwp_state = LWP_USER;
if (t->t_trapret) {
t->t_trapret = 0;
thread_lock(t);
CL_TRAPRET(t);
thread_unlock(t);
}
if (CPU->cpu_runrun)
preempt();
lwp->lwp_errno = 0; /* clear error for next time */
/*
* The thread lock must be held in order to clear sysnum and reset
* lwp_ap atomically with respect to other threads in the system that
* may be looking at the args via lwp_ap from get_syscall_args().
*/
thread_lock(t);
t->t_sysnum = 0; /* no longer in a system call */
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
/*
* In case the args were copied to the lwp, reset the
* pointer so the next syscall will have the right
* lwp_ap pointer.
*/
lwp->lwp_ap = (long *)&rp->r_rdi;
} else {
#endif
lwp->lwp_ap = NULL; /* reset on every syscall entry */
}
thread_unlock(t);
lwp->lwp_argsaved = 0;
/*
* If there was a continuing reason for post-syscall processing,
* set the t_post_sys flag for the next system call.
*/
if (repost)
t->t_post_sys = 1;
/*
* If there is a ustack registered for this lwp, and the stack rlimit
* has been altered, read in the ustack. If the saved stack rlimit
* matches the bounds of the ustack, update the ustack to reflect
* the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
* stack checking by setting the size to 0.
*/
if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
rlim64_t new_size;
caddr_t top;
stack_t stk;
struct rlimit64 rl;
mutex_enter(&p->p_lock);
new_size = p->p_stk_ctl;
top = p->p_usrstack;
(void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
mutex_exit(&p->p_lock);
if (rl.rlim_cur == RLIM64_INFINITY)
new_size = 0;
if (copyin((stack_t *)lwp->lwp_ustack, &stk,
sizeof (stack_t)) == 0 &&
(stk.ss_size == lwp->lwp_old_stk_ctl ||
stk.ss_size == 0) &&
stk.ss_sp == top - stk.ss_size) {
stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
stk.ss_size - (uintptr_t)new_size);
stk.ss_size = new_size;
(void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
sizeof (stack_t));
}
lwp->lwp_old_stk_ctl = 0;
}
}
/*
* Common tasks always done by _sys_rtt, called with interrupts disabled.
* Returns 1 if returning to userland, 0 if returning to system mode.
*/
int
sys_rtt_common(struct regs *rp)
{
kthread_t *tp;
extern void mutex_exit_critical_start();
extern long mutex_exit_critical_size;
extern void mutex_owner_running_critical_start();
extern long mutex_owner_running_critical_size;
loop:
/*
* Check if returning to user
*/
tp = CPU->cpu_thread;
if (USERMODE(rp->r_cs)) {
/*
* Check if AST pending.
*/
if (tp->t_astflag) {
/*
* Let trap() handle the AST
*/
sti();
rp->r_trapno = T_AST;
trap(rp, (caddr_t)0, CPU->cpu_id);
cli();
goto loop;
}
#if defined(__amd64)
/*
* We are done if segment registers do not need updating.
*/
if (tp->t_lwp->lwp_pcb.pcb_rupdate == 0)
return (1);
if (update_sregs(rp, tp->t_lwp)) {
/*
* 1 or more of the selectors is bad.
* Deliver a SIGSEGV.
*/
proc_t *p = ttoproc(tp);
sti();
mutex_enter(&p->p_lock);
tp->t_lwp->lwp_cursig = SIGSEGV;
mutex_exit(&p->p_lock);
psig();
tp->t_sig_check = 1;
cli();
}
tp->t_lwp->lwp_pcb.pcb_rupdate = 0;
#endif /* __amd64 */
return (1);
}
/*
* Here if we are returning to supervisor mode.
* Check for a kernel preemption request.
*/
if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
/*
* Do nothing if already in kpreempt
*/
if (!tp->t_preempt_lk) {
tp->t_preempt_lk = 1;
sti();
kpreempt(1); /* asynchronous kpreempt call */
cli();
tp->t_preempt_lk = 0;
}
}
/*
* If we interrupted the mutex_exit() critical region we must
* reset the PC back to the beginning to prevent missed wakeups
* See the comments in mutex_exit() for details.
*/
if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
mutex_exit_critical_size) {
rp->r_pc = (greg_t)mutex_exit_critical_start;
}
/*
* If we interrupted the mutex_owner_running() critical region we
* must reset the PC back to the beginning to prevent dereferencing
* of a freed thread pointer. See the comments in mutex_owner_running
* for details.
*/
if ((uintptr_t)rp->r_pc -
(uintptr_t)mutex_owner_running_critical_start <
mutex_owner_running_critical_size) {
rp->r_pc = (greg_t)mutex_owner_running_critical_start;
}
return (0);
}