int
linux_fork(struct thread *td, struct linux_fork_args *args)
{
int error;
struct proc *p2;
struct thread *td2;
#ifdef DEBUG
if (ldebug(fork))
printf(ARGS(fork, ""));
#endif
if ((error = fork1(td, RFFDG | RFPROC | RFSTOPPED, 0, &p2, NULL, 0))
!= 0)
return (error);
td->td_retval[0] = p2->p_pid;
td->td_retval[1] = 0;
error = linux_proc_init(td, td->td_retval[0], 0);
if (error)
return (error);
td2 = FIRST_THREAD_IN_PROC(p2);
/*
* Make this runnable after we are finished with it.
*/
thread_lock(td2);
TD_SET_CAN_RUN(td2);
sched_add(td2, SRQ_BORING);
thread_unlock(td2);
return (0);
}
int
linux_fork(struct thread *td, struct linux_fork_args *args)
{
struct fork_req fr;
int error;
struct proc *p2;
struct thread *td2;
#ifdef DEBUG
if (ldebug(fork))
printf(ARGS(fork, ""));
#endif
bzero(&fr, sizeof(fr));
fr.fr_flags = RFFDG | RFPROC | RFSTOPPED;
fr.fr_procp = &p2;
if ((error = fork1(td, &fr)) != 0)
return (error);
td2 = FIRST_THREAD_IN_PROC(p2);
linux_proc_init(td, td2, 0);
td->td_retval[0] = p2->p_pid;
/*
* Make this runnable after we are finished with it.
*/
thread_lock(td2);
TD_SET_CAN_RUN(td2);
sched_add(td2, SRQ_BORING);
thread_unlock(td2);
return (0);
}
/*
* Select the KSE that will be run next. From that find the thread, and
* remove it from the KSEGRP's run queue. If there is thread clustering,
* this will be what does it.
*/
struct thread *
choosethread(void)
{
struct kse *ke;
struct thread *td;
struct ksegrp *kg;
#if defined(SMP) && (defined(__i386__) || defined(__amd64__))
if (smp_active == 0 && PCPU_GET(cpuid) != 0) {
/* Shutting down, run idlethread on AP's */
td = PCPU_GET(idlethread);
ke = td->td_kse;
CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td);
ke->ke_flags |= KEF_DIDRUN;
TD_SET_RUNNING(td);
return (td);
}
#endif
retry:
ke = sched_choose();
if (ke) {
td = ke->ke_thread;
KASSERT((td->td_kse == ke), ("kse/thread mismatch"));
kg = ke->ke_ksegrp;
if (td->td_proc->p_flag & P_HADTHREADS) {
if (kg->kg_last_assigned == td) {
kg->kg_last_assigned = TAILQ_PREV(td,
threadqueue, td_runq);
}
TAILQ_REMOVE(&kg->kg_runq, td, td_runq);
kg->kg_runnable--;
}
CTR2(KTR_RUNQ, "choosethread: td=%p pri=%d",
td, td->td_priority);
} else {
/* Simulate runq_choose() having returned the idle thread */
td = PCPU_GET(idlethread);
ke = td->td_kse;
CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td);
}
ke->ke_flags |= KEF_DIDRUN;
/*
* If we are in panic, only allow system threads,
* plus the one we are running in, to be run.
*/
if (panicstr && ((td->td_proc->p_flag & P_SYSTEM) == 0 &&
(td->td_flags & TDF_INPANIC) == 0)) {
/* note that it is no longer on the run queue */
TD_SET_CAN_RUN(td);
goto retry;
}
TD_SET_RUNNING(td);
return (td);
}
int
linux_vfork(struct thread *td, struct linux_vfork_args *args)
{
int error;
struct proc *p2;
struct thread *td2;
#ifdef DEBUG
if (ldebug(vfork))
printf(ARGS(vfork, ""));
#endif
/* Exclude RFPPWAIT */
if ((error = fork1(td, RFFDG | RFPROC | RFMEM | RFSTOPPED, 0, &p2,
NULL, 0)) != 0)
return (error);
td->td_retval[0] = p2->p_pid;
error = linux_proc_init(td, td->td_retval[0], 0);
if (error)
return (error);
PROC_LOCK(p2);
p2->p_flag |= P_PPWAIT;
PROC_UNLOCK(p2);
td2 = FIRST_THREAD_IN_PROC(p2);
/*
* Make this runnable after we are finished with it.
*/
thread_lock(td2);
TD_SET_CAN_RUN(td2);
sched_add(td2, SRQ_BORING);
thread_unlock(td2);
/* wait for the children to exit, ie. emulate vfork */
PROC_LOCK(p2);
while (p2->p_flag & P_PPWAIT)
cv_wait(&p2->p_pwait, &p2->p_mtx);
PROC_UNLOCK(p2);
return (0);
}
/*
* Remove a thread from its KSEGRP's run queue.
* This in turn may remove it from a KSE if it was already assigned
* to one, possibly causing a new thread to be assigned to the KSE
* and the KSE getting a new priority.
*/
static void
remrunqueue(struct thread *td)
{
struct thread *td2, *td3;
struct ksegrp *kg;
struct kse *ke;
mtx_assert(&sched_lock, MA_OWNED);
KASSERT((TD_ON_RUNQ(td)), ("remrunqueue: Bad state on run queue"));
kg = td->td_ksegrp;
ke = td->td_kse;
CTR1(KTR_RUNQ, "remrunqueue: td%p", td);
TD_SET_CAN_RUN(td);
/*
* If it is not a threaded process, take the shortcut.
*/
if ((td->td_proc->p_flag & P_HADTHREADS) == 0) {
/* remve from sys run queue and free up a slot */
sched_rem(td);
ke->ke_state = KES_THREAD;
return;
}
td3 = TAILQ_PREV(td, threadqueue, td_runq);
TAILQ_REMOVE(&kg->kg_runq, td, td_runq);
kg->kg_runnable--;
if (ke->ke_state == KES_ONRUNQ) {
/*
* This thread has been assigned to the system run queue.
* We need to dissociate it and try assign the
* KSE to the next available thread. Then, we should
* see if we need to move the KSE in the run queues.
*/
sched_rem(td);
ke->ke_state = KES_THREAD;
td2 = kg->kg_last_assigned;
KASSERT((td2 != NULL), ("last assigned has wrong value"));
if (td2 == td)
kg->kg_last_assigned = td3;
/* slot_fill(kg); */ /* will replace it with another */
}
}
/*
* Change the priority of a thread that is on the run queue.
*/
void
adjustrunqueue( struct thread *td, int newpri)
{
struct ksegrp *kg;
struct kse *ke;
mtx_assert(&sched_lock, MA_OWNED);
KASSERT((TD_ON_RUNQ(td)), ("adjustrunqueue: Bad state on run queue"));
ke = td->td_kse;
CTR1(KTR_RUNQ, "adjustrunqueue: td%p", td);
/*
* If it is not a threaded process, take the shortcut.
*/
if ((td->td_proc->p_flag & P_HADTHREADS) == 0) {
/* We only care about the kse in the run queue. */
td->td_priority = newpri;
if (ke->ke_rqindex != (newpri / RQ_PPQ)) {
sched_rem(td);
sched_add(td, SRQ_BORING);
}
return;
}
/* It is a threaded process */
kg = td->td_ksegrp;
if (ke->ke_state == KES_ONRUNQ) {
if (kg->kg_last_assigned == td) {
kg->kg_last_assigned =
TAILQ_PREV(td, threadqueue, td_runq);
}
sched_rem(td);
}
TAILQ_REMOVE(&kg->kg_runq, td, td_runq);
kg->kg_runnable--;
TD_SET_CAN_RUN(td);
td->td_priority = newpri;
setrunqueue(td, SRQ_BORING);
}
static int
linux_clone_thread(struct thread *td, struct linux_clone_args *args)
{
struct linux_emuldata *em;
struct thread *newtd;
struct proc *p;
int error;
#ifdef DEBUG
if (ldebug(clone)) {
printf(ARGS(clone, "thread: flags %x, stack %p, parent tid: %p, "
"child tid: %p"), (unsigned)args->flags,
args->stack, args->parent_tidptr, args->child_tidptr);
}
#endif
LINUX_CTR4(clone_thread, "thread(%d) flags %x ptid %p ctid %p",
td->td_tid, (unsigned)args->flags,
args->parent_tidptr, args->child_tidptr);
if (args->flags & LINUX_CLONE_PARENT_SETTID)
if (args->parent_tidptr == NULL)
return (EINVAL);
/* Threads should be created with own stack */
if (args->stack == NULL)
return (EINVAL);
p = td->td_proc;
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(p);
error = racct_add(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
if (error != 0)
return (EPROCLIM);
}
#endif
/* Initialize our td */
error = kern_thr_alloc(p, 0, &newtd);
if (error)
goto fail;
cpu_copy_thread(newtd, td);
bzero(&newtd->td_startzero,
__rangeof(struct thread, td_startzero, td_endzero));
bcopy(&td->td_startcopy, &newtd->td_startcopy,
__rangeof(struct thread, td_startcopy, td_endcopy));
newtd->td_proc = p;
thread_cow_get(newtd, td);
/* create the emuldata */
linux_proc_init(td, newtd, args->flags);
em = em_find(newtd);
KASSERT(em != NULL, ("clone_thread: emuldata not found.\n"));
if (args->flags & LINUX_CLONE_SETTLS)
linux_set_cloned_tls(newtd, args->tls);
if (args->flags & LINUX_CLONE_CHILD_SETTID)
em->child_set_tid = args->child_tidptr;
else
em->child_set_tid = NULL;
if (args->flags & LINUX_CLONE_CHILD_CLEARTID)
em->child_clear_tid = args->child_tidptr;
else
em->child_clear_tid = NULL;
cpu_thread_clean(newtd);
linux_set_upcall_kse(newtd, PTROUT(args->stack));
PROC_LOCK(p);
p->p_flag |= P_HADTHREADS;
bcopy(p->p_comm, newtd->td_name, sizeof(newtd->td_name));
if (args->flags & LINUX_CLONE_PARENT)
thread_link(newtd, p->p_pptr);
else
thread_link(newtd, p);
thread_lock(td);
/* let the scheduler know about these things. */
sched_fork_thread(td, newtd);
thread_unlock(td);
if (P_SHOULDSTOP(p))
newtd->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
PROC_UNLOCK(p);
tidhash_add(newtd);
#ifdef DEBUG
if (ldebug(clone))
printf(ARGS(clone, "successful clone to %d, stack %p"),
(int)newtd->td_tid, args->stack);
#endif
LINUX_CTR2(clone_thread, "thread(%d) successful clone to %d",
td->td_tid, newtd->td_tid);
if (args->flags & LINUX_CLONE_PARENT_SETTID) {
error = copyout(&newtd->td_tid, args->parent_tidptr,
sizeof(newtd->td_tid));
if (error)
printf(LMSG("clone_thread: copyout failed!"));
}
/*
* Make this runnable after we are finished with it.
*/
thread_lock(newtd);
TD_SET_CAN_RUN(newtd);
sched_add(newtd, SRQ_BORING);
thread_unlock(newtd);
td->td_retval[0] = newtd->td_tid;
return (0);
fail:
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(p);
racct_sub(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
}
#endif
return (error);
}
static int
linux_clone_proc(struct thread *td, struct linux_clone_args *args)
{
struct fork_req fr;
int error, ff = RFPROC | RFSTOPPED;
struct proc *p2;
struct thread *td2;
int exit_signal;
struct linux_emuldata *em;
#ifdef DEBUG
if (ldebug(clone)) {
printf(ARGS(clone, "flags %x, stack %p, parent tid: %p, "
"child tid: %p"), (unsigned)args->flags,
args->stack, args->parent_tidptr, args->child_tidptr);
}
#endif
exit_signal = args->flags & 0x000000ff;
if (LINUX_SIG_VALID(exit_signal)) {
exit_signal = linux_to_bsd_signal(exit_signal);
} else if (exit_signal != 0)
return (EINVAL);
if (args->flags & LINUX_CLONE_VM)
ff |= RFMEM;
if (args->flags & LINUX_CLONE_SIGHAND)
ff |= RFSIGSHARE;
/*
* XXX: In Linux, sharing of fs info (chroot/cwd/umask)
* and open files is independent. In FreeBSD, its in one
* structure but in reality it does not cause any problems
* because both of these flags are usually set together.
*/
if (!(args->flags & (LINUX_CLONE_FILES | LINUX_CLONE_FS)))
ff |= RFFDG;
if (args->flags & LINUX_CLONE_PARENT_SETTID)
if (args->parent_tidptr == NULL)
return (EINVAL);
if (args->flags & LINUX_CLONE_VFORK)
ff |= RFPPWAIT;
bzero(&fr, sizeof(fr));
fr.fr_flags = ff;
fr.fr_procp = &p2;
error = fork1(td, &fr);
if (error)
return (error);
td2 = FIRST_THREAD_IN_PROC(p2);
/* create the emuldata */
linux_proc_init(td, td2, args->flags);
em = em_find(td2);
KASSERT(em != NULL, ("clone_proc: emuldata not found.\n"));
if (args->flags & LINUX_CLONE_CHILD_SETTID)
em->child_set_tid = args->child_tidptr;
else
em->child_set_tid = NULL;
if (args->flags & LINUX_CLONE_CHILD_CLEARTID)
em->child_clear_tid = args->child_tidptr;
else
em->child_clear_tid = NULL;
if (args->flags & LINUX_CLONE_PARENT_SETTID) {
error = copyout(&p2->p_pid, args->parent_tidptr,
sizeof(p2->p_pid));
if (error)
printf(LMSG("copyout failed!"));
}
PROC_LOCK(p2);
p2->p_sigparent = exit_signal;
PROC_UNLOCK(p2);
/*
* In a case of stack = NULL, we are supposed to COW calling process
* stack. This is what normal fork() does, so we just keep tf_rsp arg
* intact.
*/
linux_set_upcall_kse(td2, PTROUT(args->stack));
if (args->flags & LINUX_CLONE_SETTLS)
linux_set_cloned_tls(td2, args->tls);
/*
* If CLONE_PARENT is set, then the parent of the new process will be
* the same as that of the calling process.
*/
if (args->flags & LINUX_CLONE_PARENT) {
sx_xlock(&proctree_lock);
PROC_LOCK(p2);
proc_reparent(p2, td->td_proc->p_pptr);
PROC_UNLOCK(p2);
sx_xunlock(&proctree_lock);
}
#ifdef DEBUG
if (ldebug(clone))
printf(LMSG("clone: successful rfork to %d, "
"stack %p sig = %d"), (int)p2->p_pid, args->stack,
exit_signal);
#endif
/*
* Make this runnable after we are finished with it.
*/
thread_lock(td2);
TD_SET_CAN_RUN(td2);
sched_add(td2, SRQ_BORING);
thread_unlock(td2);
td->td_retval[0] = p2->p_pid;
return (0);
}
static int
create_thread(struct thread *td, mcontext_t *ctx,
void (*start_func)(void *), void *arg,
char *stack_base, size_t stack_size,
char *tls_base,
long *child_tid, long *parent_tid,
int flags, struct rtprio *rtp)
{
stack_t stack;
struct thread *newtd;
struct proc *p;
int error;
p = td->td_proc;
/* Have race condition but it is cheap. */
if (p->p_numthreads >= max_threads_per_proc) {
++max_threads_hits;
return (EPROCLIM);
}
if (rtp != NULL) {
switch(rtp->type) {
case RTP_PRIO_REALTIME:
case RTP_PRIO_FIFO:
/* Only root can set scheduler policy */
if (priv_check(td, PRIV_SCHED_SETPOLICY) != 0)
return (EPERM);
if (rtp->prio > RTP_PRIO_MAX)
return (EINVAL);
break;
case RTP_PRIO_NORMAL:
rtp->prio = 0;
break;
default:
return (EINVAL);
}
}
#ifdef RACCT
PROC_LOCK(td->td_proc);
error = racct_add(p, RACCT_NTHR, 1);
PROC_UNLOCK(td->td_proc);
if (error != 0)
return (EPROCLIM);
#endif
/* Initialize our td */
newtd = thread_alloc(0);
if (newtd == NULL) {
error = ENOMEM;
goto fail;
}
cpu_set_upcall(newtd, td);
/*
* Try the copyout as soon as we allocate the td so we don't
* have to tear things down in a failure case below.
* Here we copy out tid to two places, one for child and one
* for parent, because pthread can create a detached thread,
* if parent wants to safely access child tid, it has to provide
* its storage, because child thread may exit quickly and
* memory is freed before parent thread can access it.
*/
if ((child_tid != NULL &&
suword_lwpid(child_tid, newtd->td_tid)) ||
(parent_tid != NULL &&
suword_lwpid(parent_tid, newtd->td_tid))) {
thread_free(newtd);
error = EFAULT;
goto fail;
}
bzero(&newtd->td_startzero,
__rangeof(struct thread, td_startzero, td_endzero));
bcopy(&td->td_startcopy, &newtd->td_startcopy,
__rangeof(struct thread, td_startcopy, td_endcopy));
newtd->td_proc = td->td_proc;
newtd->td_ucred = crhold(td->td_ucred);
if (ctx != NULL) { /* old way to set user context */
error = set_mcontext(newtd, ctx);
if (error != 0) {
thread_free(newtd);
crfree(td->td_ucred);
goto fail;
}
} else {
/* Set up our machine context. */
stack.ss_sp = stack_base;
stack.ss_size = stack_size;
/* Set upcall address to user thread entry function. */
cpu_set_upcall_kse(newtd, start_func, arg, &stack);
/* Setup user TLS address and TLS pointer register. */
error = cpu_set_user_tls(newtd, tls_base);
if (error != 0) {
thread_free(newtd);
crfree(td->td_ucred);
goto fail;
}
}
PROC_LOCK(td->td_proc);
td->td_proc->p_flag |= P_HADTHREADS;
thread_link(newtd, p);
bcopy(p->p_comm, newtd->td_name, sizeof(newtd->td_name));
thread_lock(td);
/* let the scheduler know about these things. */
sched_fork_thread(td, newtd);
thread_unlock(td);
if (P_SHOULDSTOP(p))
newtd->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
PROC_UNLOCK(p);
tidhash_add(newtd);
thread_lock(newtd);
if (rtp != NULL) {
if (!(td->td_pri_class == PRI_TIMESHARE &&
rtp->type == RTP_PRIO_NORMAL)) {
rtp_to_pri(rtp, newtd);
sched_prio(newtd, newtd->td_user_pri);
} /* ignore timesharing class */
}
TD_SET_CAN_RUN(newtd);
sched_add(newtd, SRQ_BORING);
thread_unlock(newtd);
return (0);
fail:
#ifdef RACCT
PROC_LOCK(p);
racct_sub(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
#endif
return (error);
}
int
thread_create(struct thread *td, struct rtprio *rtp,
int (*initialize_thread)(struct thread *, void *), void *thunk)
{
struct thread *newtd;
struct proc *p;
int error;
p = td->td_proc;
if (rtp != NULL) {
switch(rtp->type) {
case RTP_PRIO_REALTIME:
case RTP_PRIO_FIFO:
/* Only root can set scheduler policy */
if (priv_check(td, PRIV_SCHED_SETPOLICY) != 0)
return (EPERM);
if (rtp->prio > RTP_PRIO_MAX)
return (EINVAL);
break;
case RTP_PRIO_NORMAL:
rtp->prio = 0;
break;
default:
return (EINVAL);
}
}
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(p);
error = racct_add(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
if (error != 0)
return (EPROCLIM);
}
#endif
/* Initialize our td */
error = kern_thr_alloc(p, 0, &newtd);
if (error)
goto fail;
cpu_set_upcall(newtd, td);
bzero(&newtd->td_startzero,
__rangeof(struct thread, td_startzero, td_endzero));
bcopy(&td->td_startcopy, &newtd->td_startcopy,
__rangeof(struct thread, td_startcopy, td_endcopy));
newtd->td_proc = td->td_proc;
thread_cow_get(newtd, td);
error = initialize_thread(newtd, thunk);
if (error != 0) {
thread_cow_free(newtd);
thread_free(newtd);
goto fail;
}
PROC_LOCK(p);
p->p_flag |= P_HADTHREADS;
thread_link(newtd, p);
bcopy(p->p_comm, newtd->td_name, sizeof(newtd->td_name));
newtd->td_pax = p->p_pax;
thread_lock(td);
/* let the scheduler know about these things. */
sched_fork_thread(td, newtd);
thread_unlock(td);
if (P_SHOULDSTOP(p))
newtd->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
if (p->p_flag2 & P2_LWP_EVENTS)
newtd->td_dbgflags |= TDB_BORN;
/*
* Copy the existing thread VM policy into the new thread.
*/
vm_domain_policy_localcopy(&newtd->td_vm_dom_policy,
&td->td_vm_dom_policy);
PROC_UNLOCK(p);
tidhash_add(newtd);
thread_lock(newtd);
if (rtp != NULL) {
if (!(td->td_pri_class == PRI_TIMESHARE &&
rtp->type == RTP_PRIO_NORMAL)) {
rtp_to_pri(rtp, newtd);
sched_prio(newtd, newtd->td_user_pri);
} /* ignore timesharing class */
}
TD_SET_CAN_RUN(newtd);
sched_add(newtd, SRQ_BORING);
thread_unlock(newtd);
return (0);
fail:
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(p);
racct_sub(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
}
#endif
return (error);
}
int
linux_clone(struct thread *td, struct linux_clone_args *args)
{
int error, ff = RFPROC | RFSTOPPED;
struct proc *p2;
struct thread *td2;
int exit_signal;
struct linux_emuldata *em;
#ifdef DEBUG
if (ldebug(clone)) {
printf(ARGS(clone, "flags %x, stack %p, parent tid: %p, "
"child tid: %p"), (unsigned)args->flags,
args->stack, args->parent_tidptr, args->child_tidptr);
}
#endif
exit_signal = args->flags & 0x000000ff;
if (LINUX_SIG_VALID(exit_signal)) {
if (exit_signal <= LINUX_SIGTBLSZ)
exit_signal =
linux_to_bsd_signal[_SIG_IDX(exit_signal)];
} else if (exit_signal != 0)
return (EINVAL);
if (args->flags & LINUX_CLONE_VM)
ff |= RFMEM;
if (args->flags & LINUX_CLONE_SIGHAND)
ff |= RFSIGSHARE;
/*
* XXX: In Linux, sharing of fs info (chroot/cwd/umask)
* and open files is independant. In FreeBSD, its in one
* structure but in reality it does not cause any problems
* because both of these flags are usually set together.
*/
if (!(args->flags & (LINUX_CLONE_FILES | LINUX_CLONE_FS)))
ff |= RFFDG;
/*
* Attempt to detect when linux_clone(2) is used for creating
* kernel threads. Unfortunately despite the existence of the
* CLONE_THREAD flag, version of linuxthreads package used in
* most popular distros as of beginning of 2005 doesn't make
* any use of it. Therefore, this detection relies on
* empirical observation that linuxthreads sets certain
* combination of flags, so that we can make more or less
* precise detection and notify the FreeBSD kernel that several
* processes are in fact part of the same threading group, so
* that special treatment is necessary for signal delivery
* between those processes and fd locking.
*/
if ((args->flags & 0xffffff00) == LINUX_THREADING_FLAGS)
ff |= RFTHREAD;
if (args->flags & LINUX_CLONE_PARENT_SETTID)
if (args->parent_tidptr == NULL)
return (EINVAL);
error = fork1(td, ff, 0, &p2, NULL, 0);
if (error)
return (error);
if (args->flags & (LINUX_CLONE_PARENT | LINUX_CLONE_THREAD)) {
sx_xlock(&proctree_lock);
PROC_LOCK(p2);
proc_reparent(p2, td->td_proc->p_pptr);
PROC_UNLOCK(p2);
sx_xunlock(&proctree_lock);
}
/* create the emuldata */
error = linux_proc_init(td, p2->p_pid, args->flags);
/* reference it - no need to check this */
em = em_find(p2, EMUL_DOLOCK);
KASSERT(em != NULL, ("clone: emuldata not found."));
/* and adjust it */
if (args->flags & LINUX_CLONE_THREAD) {
#ifdef notyet
PROC_LOCK(p2);
p2->p_pgrp = td->td_proc->p_pgrp;
PROC_UNLOCK(p2);
#endif
exit_signal = 0;
}
if (args->flags & LINUX_CLONE_CHILD_SETTID)
em->child_set_tid = args->child_tidptr;
else
em->child_set_tid = NULL;
if (args->flags & LINUX_CLONE_CHILD_CLEARTID)
em->child_clear_tid = args->child_tidptr;
else
em->child_clear_tid = NULL;
EMUL_UNLOCK(&emul_lock);
if (args->flags & LINUX_CLONE_PARENT_SETTID) {
error = copyout(&p2->p_pid, args->parent_tidptr,
sizeof(p2->p_pid));
if (error)
printf(LMSG("copyout failed!"));
}
PROC_LOCK(p2);
p2->p_sigparent = exit_signal;
PROC_UNLOCK(p2);
td2 = FIRST_THREAD_IN_PROC(p2);
/*
* In a case of stack = NULL, we are supposed to COW calling process
* stack. This is what normal fork() does, so we just keep tf_rsp arg
* intact.
*/
if (args->stack)
linux_set_upcall_kse(td2, PTROUT(args->stack));
if (args->flags & LINUX_CLONE_SETTLS)
linux_set_cloned_tls(td2, args->tls);
#ifdef DEBUG
if (ldebug(clone))
printf(LMSG("clone: successful rfork to %d, "
"stack %p sig = %d"), (int)p2->p_pid, args->stack,
exit_signal);
#endif
if (args->flags & LINUX_CLONE_VFORK) {
PROC_LOCK(p2);
p2->p_flag |= P_PPWAIT;
PROC_UNLOCK(p2);
}
/*
* Make this runnable after we are finished with it.
*/
thread_lock(td2);
TD_SET_CAN_RUN(td2);
sched_add(td2, SRQ_BORING);
thread_unlock(td2);
td->td_retval[0] = p2->p_pid;
td->td_retval[1] = 0;
if (args->flags & LINUX_CLONE_VFORK) {
/* wait for the children to exit, ie. emulate vfork */
PROC_LOCK(p2);
while (p2->p_flag & P_PPWAIT)
cv_wait(&p2->p_pwait, &p2->p_mtx);
PROC_UNLOCK(p2);
}
return (0);
}
static void
do_fork(struct thread *td, struct fork_req *fr, struct proc *p2, struct thread *td2,
struct vmspace *vm2, struct file *fp_procdesc)
{
struct proc *p1, *pptr;
int trypid;
struct filedesc *fd;
struct filedesc_to_leader *fdtol;
struct sigacts *newsigacts;
sx_assert(&proctree_lock, SX_SLOCKED);
sx_assert(&allproc_lock, SX_XLOCKED);
p1 = td->td_proc;
trypid = fork_findpid(fr->fr_flags);
sx_sunlock(&proctree_lock);
p2->p_state = PRS_NEW; /* protect against others */
p2->p_pid = trypid;
AUDIT_ARG_PID(p2->p_pid);
LIST_INSERT_HEAD(&allproc, p2, p_list);
allproc_gen++;
LIST_INSERT_HEAD(PIDHASH(p2->p_pid), p2, p_hash);
tidhash_add(td2);
PROC_LOCK(p2);
PROC_LOCK(p1);
sx_xunlock(&allproc_lock);
bcopy(&p1->p_startcopy, &p2->p_startcopy,
__rangeof(struct proc, p_startcopy, p_endcopy));
pargs_hold(p2->p_args);
PROC_UNLOCK(p1);
bzero(&p2->p_startzero,
__rangeof(struct proc, p_startzero, p_endzero));
/* Tell the prison that we exist. */
prison_proc_hold(p2->p_ucred->cr_prison);
PROC_UNLOCK(p2);
/*
* Malloc things while we don't hold any locks.
*/
if (fr->fr_flags & RFSIGSHARE)
newsigacts = NULL;
else
newsigacts = sigacts_alloc();
/*
* Copy filedesc.
*/
if (fr->fr_flags & RFCFDG) {
fd = fdinit(p1->p_fd, false);
fdtol = NULL;
} else if (fr->fr_flags & RFFDG) {
fd = fdcopy(p1->p_fd);
fdtol = NULL;
} else {
fd = fdshare(p1->p_fd);
if (p1->p_fdtol == NULL)
p1->p_fdtol = filedesc_to_leader_alloc(NULL, NULL,
p1->p_leader);
if ((fr->fr_flags & RFTHREAD) != 0) {
/*
* Shared file descriptor table, and shared
* process leaders.
*/
fdtol = p1->p_fdtol;
FILEDESC_XLOCK(p1->p_fd);
fdtol->fdl_refcount++;
FILEDESC_XUNLOCK(p1->p_fd);
} else {
/*
* Shared file descriptor table, and different
* process leaders.
*/
fdtol = filedesc_to_leader_alloc(p1->p_fdtol,
p1->p_fd, p2);
}
}
/*
* Make a proc table entry for the new process.
* Start by zeroing the section of proc that is zero-initialized,
* then copy the section that is copied directly from the parent.
*/
PROC_LOCK(p2);
PROC_LOCK(p1);
bzero(&td2->td_startzero,
__rangeof(struct thread, td_startzero, td_endzero));
bcopy(&td->td_startcopy, &td2->td_startcopy,
__rangeof(struct thread, td_startcopy, td_endcopy));
bcopy(&p2->p_comm, &td2->td_name, sizeof(td2->td_name));
td2->td_sigstk = td->td_sigstk;
td2->td_flags = TDF_INMEM;
td2->td_lend_user_pri = PRI_MAX;
#ifdef VIMAGE
td2->td_vnet = NULL;
td2->td_vnet_lpush = NULL;
#endif
/*
* Allow the scheduler to initialize the child.
*/
thread_lock(td);
sched_fork(td, td2);
thread_unlock(td);
/*
* Duplicate sub-structures as needed.
* Increase reference counts on shared objects.
*/
p2->p_flag = P_INMEM;
p2->p_flag2 = p1->p_flag2 & (P2_NOTRACE | P2_NOTRACE_EXEC | P2_TRAPCAP);
p2->p_swtick = ticks;
if (p1->p_flag & P_PROFIL)
startprofclock(p2);
/*
* Whilst the proc lock is held, copy the VM domain data out
* using the VM domain method.
*/
vm_domain_policy_init(&p2->p_vm_dom_policy);
vm_domain_policy_localcopy(&p2->p_vm_dom_policy,
&p1->p_vm_dom_policy);
if (fr->fr_flags & RFSIGSHARE) {
p2->p_sigacts = sigacts_hold(p1->p_sigacts);
} else {
sigacts_copy(newsigacts, p1->p_sigacts);
p2->p_sigacts = newsigacts;
}
if (fr->fr_flags & RFTSIGZMB)
p2->p_sigparent = RFTSIGNUM(fr->fr_flags);
else if (fr->fr_flags & RFLINUXTHPN)
p2->p_sigparent = SIGUSR1;
else
p2->p_sigparent = SIGCHLD;
p2->p_textvp = p1->p_textvp;
p2->p_fd = fd;
p2->p_fdtol = fdtol;
if (p1->p_flag2 & P2_INHERIT_PROTECTED) {
p2->p_flag |= P_PROTECTED;
p2->p_flag2 |= P2_INHERIT_PROTECTED;
}
/*
* p_limit is copy-on-write. Bump its refcount.
*/
lim_fork(p1, p2);
thread_cow_get_proc(td2, p2);
pstats_fork(p1->p_stats, p2->p_stats);
PROC_UNLOCK(p1);
PROC_UNLOCK(p2);
/* Bump references to the text vnode (for procfs). */
if (p2->p_textvp)
vrefact(p2->p_textvp);
/*
* Set up linkage for kernel based threading.
*/
if ((fr->fr_flags & RFTHREAD) != 0) {
mtx_lock(&ppeers_lock);
p2->p_peers = p1->p_peers;
p1->p_peers = p2;
p2->p_leader = p1->p_leader;
mtx_unlock(&ppeers_lock);
PROC_LOCK(p1->p_leader);
if ((p1->p_leader->p_flag & P_WEXIT) != 0) {
PROC_UNLOCK(p1->p_leader);
/*
* The task leader is exiting, so process p1 is
* going to be killed shortly. Since p1 obviously
* isn't dead yet, we know that the leader is either
* sending SIGKILL's to all the processes in this
* task or is sleeping waiting for all the peers to
* exit. We let p1 complete the fork, but we need
* to go ahead and kill the new process p2 since
* the task leader may not get a chance to send
* SIGKILL to it. We leave it on the list so that
* the task leader will wait for this new process
* to commit suicide.
*/
PROC_LOCK(p2);
kern_psignal(p2, SIGKILL);
PROC_UNLOCK(p2);
} else
PROC_UNLOCK(p1->p_leader);
} else {
p2->p_peers = NULL;
p2->p_leader = p2;
}
sx_xlock(&proctree_lock);
PGRP_LOCK(p1->p_pgrp);
PROC_LOCK(p2);
PROC_LOCK(p1);
/*
* Preserve some more flags in subprocess. P_PROFIL has already
* been preserved.
*/
p2->p_flag |= p1->p_flag & P_SUGID;
td2->td_pflags |= (td->td_pflags & TDP_ALTSTACK) | TDP_FORKING;
SESS_LOCK(p1->p_session);
if (p1->p_session->s_ttyvp != NULL && p1->p_flag & P_CONTROLT)
p2->p_flag |= P_CONTROLT;
SESS_UNLOCK(p1->p_session);
if (fr->fr_flags & RFPPWAIT)
p2->p_flag |= P_PPWAIT;
p2->p_pgrp = p1->p_pgrp;
LIST_INSERT_AFTER(p1, p2, p_pglist);
PGRP_UNLOCK(p1->p_pgrp);
LIST_INIT(&p2->p_children);
LIST_INIT(&p2->p_orphans);
callout_init_mtx(&p2->p_itcallout, &p2->p_mtx, 0);
/*
* If PF_FORK is set, the child process inherits the
* procfs ioctl flags from its parent.
*/
if (p1->p_pfsflags & PF_FORK) {
p2->p_stops = p1->p_stops;
p2->p_pfsflags = p1->p_pfsflags;
}
/*
* This begins the section where we must prevent the parent
* from being swapped.
*/
_PHOLD(p1);
PROC_UNLOCK(p1);
/*
* Attach the new process to its parent.
*
* If RFNOWAIT is set, the newly created process becomes a child
* of init. This effectively disassociates the child from the
* parent.
*/
if ((fr->fr_flags & RFNOWAIT) != 0) {
pptr = p1->p_reaper;
p2->p_reaper = pptr;
} else {
p2->p_reaper = (p1->p_treeflag & P_TREE_REAPER) != 0 ?
p1 : p1->p_reaper;
pptr = p1;
}
p2->p_pptr = pptr;
LIST_INSERT_HEAD(&pptr->p_children, p2, p_sibling);
LIST_INIT(&p2->p_reaplist);
LIST_INSERT_HEAD(&p2->p_reaper->p_reaplist, p2, p_reapsibling);
if (p2->p_reaper == p1)
p2->p_reapsubtree = p2->p_pid;
sx_xunlock(&proctree_lock);
/* Inform accounting that we have forked. */
p2->p_acflag = AFORK;
PROC_UNLOCK(p2);
#ifdef KTRACE
ktrprocfork(p1, p2);
#endif
/*
* Finish creating the child process. It will return via a different
* execution path later. (ie: directly into user mode)
*/
vm_forkproc(td, p2, td2, vm2, fr->fr_flags);
if (fr->fr_flags == (RFFDG | RFPROC)) {
VM_CNT_INC(v_forks);
VM_CNT_ADD(v_forkpages, p2->p_vmspace->vm_dsize +
p2->p_vmspace->vm_ssize);
} else if (fr->fr_flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) {
VM_CNT_INC(v_vforks);
VM_CNT_ADD(v_vforkpages, p2->p_vmspace->vm_dsize +
p2->p_vmspace->vm_ssize);
} else if (p1 == &proc0) {
VM_CNT_INC(v_kthreads);
VM_CNT_ADD(v_kthreadpages, p2->p_vmspace->vm_dsize +
p2->p_vmspace->vm_ssize);
} else {
VM_CNT_INC(v_rforks);
VM_CNT_ADD(v_rforkpages, p2->p_vmspace->vm_dsize +
p2->p_vmspace->vm_ssize);
}
/*
* Associate the process descriptor with the process before anything
* can happen that might cause that process to need the descriptor.
* However, don't do this until after fork(2) can no longer fail.
*/
if (fr->fr_flags & RFPROCDESC)
procdesc_new(p2, fr->fr_pd_flags);
/*
* Both processes are set up, now check if any loadable modules want
* to adjust anything.
*/
EVENTHANDLER_INVOKE(process_fork, p1, p2, fr->fr_flags);
/*
* Set the child start time and mark the process as being complete.
*/
PROC_LOCK(p2);
PROC_LOCK(p1);
microuptime(&p2->p_stats->p_start);
PROC_SLOCK(p2);
p2->p_state = PRS_NORMAL;
PROC_SUNLOCK(p2);
#ifdef KDTRACE_HOOKS
/*
* Tell the DTrace fasttrap provider about the new process so that any
* tracepoints inherited from the parent can be removed. We have to do
* this only after p_state is PRS_NORMAL since the fasttrap module will
* use pfind() later on.
*/
if ((fr->fr_flags & RFMEM) == 0 && dtrace_fasttrap_fork)
dtrace_fasttrap_fork(p1, p2);
#endif
/*
* Hold the process so that it cannot exit after we make it runnable,
* but before we wait for the debugger.
*/
_PHOLD(p2);
if (p1->p_ptevents & PTRACE_FORK) {
/*
* Arrange for debugger to receive the fork event.
*
* We can report PL_FLAG_FORKED regardless of
* P_FOLLOWFORK settings, but it does not make a sense
* for runaway child.
*/
td->td_dbgflags |= TDB_FORK;
td->td_dbg_forked = p2->p_pid;
td2->td_dbgflags |= TDB_STOPATFORK;
}
if (fr->fr_flags & RFPPWAIT) {
td->td_pflags |= TDP_RFPPWAIT;
td->td_rfppwait_p = p2;
td->td_dbgflags |= TDB_VFORK;
}
PROC_UNLOCK(p2);
/*
* Now can be swapped.
*/
_PRELE(p1);
PROC_UNLOCK(p1);
/*
* Tell any interested parties about the new process.
*/
knote_fork(p1->p_klist, p2->p_pid);
SDT_PROBE3(proc, , , create, p2, p1, fr->fr_flags);
if (fr->fr_flags & RFPROCDESC) {
procdesc_finit(p2->p_procdesc, fp_procdesc);
fdrop(fp_procdesc, td);
}
if ((fr->fr_flags & RFSTOPPED) == 0) {
/*
* If RFSTOPPED not requested, make child runnable and
* add to run queue.
*/
thread_lock(td2);
TD_SET_CAN_RUN(td2);
sched_add(td2, SRQ_BORING);
thread_unlock(td2);
if (fr->fr_pidp != NULL)
*fr->fr_pidp = p2->p_pid;
} else {
*fr->fr_procp = p2;
}
PROC_LOCK(p2);
/*
* Wait until debugger is attached to child.
*/
while (td2->td_proc == p2 && (td2->td_dbgflags & TDB_STOPATFORK) != 0)
cv_wait(&p2->p_dbgwait, &p2->p_mtx);
_PRELE(p2);
racct_proc_fork_done(p2);
PROC_UNLOCK(p2);
}
int
thread_create(struct thread *td, struct rtprio *rtp,
int (*initialize_thread)(struct thread *, void *), void *thunk)
{
struct thread *newtd;
struct proc *p;
int error;
p = td->td_proc;
if (rtp != NULL) {
switch(rtp->type) {
case RTP_PRIO_REALTIME:
case RTP_PRIO_FIFO:
/* Only root can set scheduler policy */
if (priv_check(td, PRIV_SCHED_SETPOLICY) != 0)
return (EPERM);
if (rtp->prio > RTP_PRIO_MAX)
return (EINVAL);
break;
case RTP_PRIO_NORMAL:
rtp->prio = 0;
break;
default:
return (EINVAL);
}
}
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(p);
error = racct_add(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
if (error != 0)
return (EPROCLIM);
}
#endif
/* Initialize our td */
error = kern_thr_alloc(p, 0, &newtd);
if (error)
goto fail;
cpu_copy_thread(newtd, td);
bzero(&newtd->td_startzero,
__rangeof(struct thread, td_startzero, td_endzero));
bcopy(&td->td_startcopy, &newtd->td_startcopy,
__rangeof(struct thread, td_startcopy, td_endcopy));
newtd->td_proc = td->td_proc;
newtd->td_rb_list = newtd->td_rbp_list = newtd->td_rb_inact = 0;
thread_cow_get(newtd, td);
error = initialize_thread(newtd, thunk);
if (error != 0) {
thread_cow_free(newtd);
thread_free(newtd);
goto fail;
}
PROC_LOCK(p);
p->p_flag |= P_HADTHREADS;
thread_link(newtd, p);
bcopy(p->p_comm, newtd->td_name, sizeof(newtd->td_name));
thread_lock(td);
/* let the scheduler know about these things. */
sched_fork_thread(td, newtd);
thread_unlock(td);
if (P_SHOULDSTOP(p))
newtd->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
if (p->p_ptevents & PTRACE_LWP)
newtd->td_dbgflags |= TDB_BORN;
PROC_UNLOCK(p);
#ifdef HWPMC_HOOKS
if (PMC_PROC_IS_USING_PMCS(p))
PMC_CALL_HOOK(newtd, PMC_FN_THR_CREATE, NULL);
else if (PMC_SYSTEM_SAMPLING_ACTIVE())
PMC_CALL_HOOK_UNLOCKED(newtd, PMC_FN_THR_CREATE_LOG, NULL);
#endif
tidhash_add(newtd);
thread_lock(newtd);
if (rtp != NULL) {
if (!(td->td_pri_class == PRI_TIMESHARE &&
rtp->type == RTP_PRIO_NORMAL)) {
rtp_to_pri(rtp, newtd);
sched_prio(newtd, newtd->td_user_pri);
} /* ignore timesharing class */
}
TD_SET_CAN_RUN(newtd);
sched_add(newtd, SRQ_BORING);
thread_unlock(newtd);
return (0);
fail:
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(p);
racct_sub(p, RACCT_NTHR, 1);
PROC_UNLOCK(p);
}
#endif
return (error);
}
/*
* Create a kernel process/thread/whatever. It shares its address space
* with proc0 - ie: kernel only.
*
* func is the function to start.
* arg is the parameter to pass to function on first startup.
* newpp is the return value pointing to the thread's struct proc.
* flags are flags to fork1 (in unistd.h)
* fmt and following will be *printf'd into (*newpp)->p_comm (for ps, etc.).
*/
int
kproc_create(void (*func)(void *), void *arg,
struct proc **newpp, int flags, int pages, const char *fmt, ...)
{
struct fork_req fr;
int error;
va_list ap;
struct thread *td;
struct proc *p2;
if (!proc0.p_stats)
panic("kproc_create called too soon");
bzero(&fr, sizeof(fr));
fr.fr_flags = RFMEM | RFFDG | RFPROC | RFSTOPPED | flags;
fr.fr_pages = pages;
fr.fr_procp = &p2;
error = fork1(&thread0, &fr);
if (error)
return error;
/* save a global descriptor, if desired */
if (newpp != NULL)
*newpp = p2;
/* this is a non-swapped system process */
PROC_LOCK(p2);
td = FIRST_THREAD_IN_PROC(p2);
p2->p_flag |= P_SYSTEM | P_KTHREAD;
td->td_pflags |= TDP_KTHREAD;
mtx_lock(&p2->p_sigacts->ps_mtx);
p2->p_sigacts->ps_flag |= PS_NOCLDWAIT;
mtx_unlock(&p2->p_sigacts->ps_mtx);
PROC_UNLOCK(p2);
/* set up arg0 for 'ps', et al */
va_start(ap, fmt);
vsnprintf(p2->p_comm, sizeof(p2->p_comm), fmt, ap);
va_end(ap);
/* set up arg0 for 'ps', et al */
va_start(ap, fmt);
vsnprintf(td->td_name, sizeof(td->td_name), fmt, ap);
va_end(ap);
#ifdef KTR
sched_clear_tdname(td);
#endif
/* call the processes' main()... */
cpu_set_fork_handler(td, func, arg);
/* Avoid inheriting affinity from a random parent. */
cpuset_setthread(td->td_tid, cpuset_root);
thread_lock(td);
TD_SET_CAN_RUN(td);
sched_prio(td, PVM);
sched_user_prio(td, PUSER);
/* Delay putting it on the run queue until now. */
if (!(flags & RFSTOPPED))
sched_add(td, SRQ_BORING);
thread_unlock(td);
return 0;
}
/*
* Create a kernel thread. It shares its address space
* with proc0 - ie: kernel only.
*
* func is the function to start.
* arg is the parameter to pass to function on first startup.
* newtdp is the return value pointing to the thread's struct thread.
* ** XXX fix this --> flags are flags to fork1 (in unistd.h)
* fmt and following will be *printf'd into (*newtd)->td_name (for ps, etc.).
*/
int
kthread_add(void (*func)(void *), void *arg, struct proc *p,
struct thread **newtdp, int flags, int pages, const char *fmt, ...)
{
va_list ap;
struct thread *newtd, *oldtd;
if (!proc0.p_stats)
panic("kthread_add called too soon");
/* If no process supplied, put it on proc0 */
if (p == NULL)
p = &proc0;
/* Initialize our new td */
newtd = thread_alloc(pages);
if (newtd == NULL)
return (ENOMEM);
PROC_LOCK(p);
oldtd = FIRST_THREAD_IN_PROC(p);
bzero(&newtd->td_startzero,
__rangeof(struct thread, td_startzero, td_endzero));
bcopy(&oldtd->td_startcopy, &newtd->td_startcopy,
__rangeof(struct thread, td_startcopy, td_endcopy));
/* set up arg0 for 'ps', et al */
va_start(ap, fmt);
vsnprintf(newtd->td_name, sizeof(newtd->td_name), fmt, ap);
va_end(ap);
newtd->td_proc = p; /* needed for cpu_set_upcall */
/* XXX optimise this probably? */
/* On x86 (and probably the others too) it is way too full of junk */
/* Needs a better name */
cpu_set_upcall(newtd, oldtd);
/* put the designated function(arg) as the resume context */
cpu_set_fork_handler(newtd, func, arg);
newtd->td_pflags |= TDP_KTHREAD;
thread_cow_get_proc(newtd, p);
/* this code almost the same as create_thread() in kern_thr.c */
p->p_flag |= P_HADTHREADS;
thread_link(newtd, p);
thread_lock(oldtd);
/* let the scheduler know about these things. */
sched_fork_thread(oldtd, newtd);
TD_SET_CAN_RUN(newtd);
thread_unlock(oldtd);
PROC_UNLOCK(p);
tidhash_add(newtd);
/* Avoid inheriting affinity from a random parent. */
cpuset_setthread(newtd->td_tid, cpuset_root);
/* Delay putting it on the run queue until now. */
if (!(flags & RFSTOPPED)) {
thread_lock(newtd);
sched_add(newtd, SRQ_BORING);
thread_unlock(newtd);
}
if (newtdp)
*newtdp = newtd;
return 0;
}
