/*
* Acquire a page's "shared/exclusive" lock, but never block.
* Returns 1 on success, 0 on failure.
*/
int
page_trylock(page_t *pp, se_t se)
{
kmutex_t *pse = PAGE_SE_MUTEX(pp);
mutex_enter(pse);
if (pp->p_selock & SE_EWANTED || PP_RETIRED(pp) ||
(se == SE_SHARED && PP_PR_NOSHARE(pp))) {
/*
* Fail if a thread wants exclusive access and page is
* retired, if the page is slated for retirement, or a
* share lock is requested.
*/
mutex_exit(pse);
VM_STAT_ADD(page_trylock_failed);
return (0);
}
if (se == SE_EXCL) {
if (pp->p_selock == 0) {
THREAD_KPRI_REQUEST();
pp->p_selock = SE_WRITER;
mutex_exit(pse);
return (1);
}
} else {
if (pp->p_selock >= 0) {
pp->p_selock += SE_READER;
mutex_exit(pse);
return (1);
}
}
mutex_exit(pse);
return (0);
}
/*ARGSUSED*/
int64_t
loadable_syscall(
long a0, long a1, long a2, long a3,
long a4, long a5, long a6, long a7)
{
klwp_t *lwp = ttolwp(curthread);
int64_t rval;
struct sysent *callp;
struct sysent *se = LWP_GETSYSENT(lwp);
krwlock_t *module_lock;
int code, error = 0;
int64_t (*sy_call)();
code = curthread->t_sysnum;
callp = se + code;
/*
* Try to autoload the system call if necessary
*/
module_lock = lock_syscall(se, code);
THREAD_KPRI_RELEASE(); /* drop priority given by rw_enter */
/*
* we've locked either the loaded syscall or nosys
*/
if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
if (callp->sy_flags & SE_ARGC) {
sy_call = (int64_t (*)())callp->sy_call;
rval = (*sy_call)(a0, a1, a2, a3, a4, a5);
} else
rval = syscall_ap();
} else {
#endif
/*
* Now that it's loaded, make sure enough args were copied.
*/
if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg))
error = EFAULT;
if (error) {
rval = set_errno(error);
} else if (callp->sy_flags & SE_ARGC) {
sy_call = (int64_t (*)())callp->sy_call;
rval = (*sy_call)(lwp->lwp_ap[0], lwp->lwp_ap[1],
lwp->lwp_ap[2], lwp->lwp_ap[3], lwp->lwp_ap[4],
lwp->lwp_ap[5]);
} else
rval = syscall_ap();
}
THREAD_KPRI_REQUEST(); /* regain priority from read lock */
rw_exit(module_lock);
return (rval);
}
/*
* lxpr_lock():
*
* Lookup process from pid and return with p_plock and P_PR_LOCK held.
*/
proc_t *
lxpr_lock(pid_t pid)
{
proc_t *p;
kmutex_t *mp;
ASSERT(!MUTEX_HELD(&pidlock));
for (;;) {
mutex_enter(&pidlock);
/*
* If the pid is 1, we really want the zone's init process
*/
p = prfind((pid == 1) ?
curproc->p_zone->zone_proc_initpid : pid);
if (p == NULL || p->p_stat == SIDL) {
mutex_exit(&pidlock);
return (NULL);
}
/*
* p_lock is persistent, but p itself is not -- it could
* vanish during cv_wait(). Load p->p_lock now so we can
* drop it after cv_wait() without referencing p.
*/
mp = &p->p_lock;
mutex_enter(mp);
mutex_exit(&pidlock);
if (p->p_flag & SEXITING) {
/*
* This process is exiting -- let it go.
*/
mutex_exit(mp);
return (NULL);
}
if (!(p->p_proc_flag & P_PR_LOCK))
break;
cv_wait(&pr_pid_cv[p->p_slot], mp);
mutex_exit(mp);
}
p->p_proc_flag |= P_PR_LOCK;
THREAD_KPRI_REQUEST();
return (p);
}
/*
* Release the page's "shared/exclusive" lock and wake up anyone
* who might be waiting for it.
*/
void
page_unlock(page_t *pp)
{
kmutex_t *pse = PAGE_SE_MUTEX(pp);
selock_t old;
mutex_enter(pse);
old = pp->p_selock;
if ((old & ~SE_EWANTED) == SE_READER) {
pp->p_selock = old & ~SE_READER;
if (CV_HAS_WAITERS(&pp->p_cv))
cv_broadcast(&pp->p_cv);
} else if ((old & ~SE_EWANTED) == SE_DELETED) {
panic("page_unlock: page %p is deleted", pp);
} else if (old < 0) {
THREAD_KPRI_RELEASE();
pp->p_selock &= SE_EWANTED;
if (CV_HAS_WAITERS(&pp->p_cv))
cv_broadcast(&pp->p_cv);
} else if ((old & ~SE_EWANTED) > SE_READER) {
pp->p_selock = old - SE_READER;
} else {
panic("page_unlock: page %p is not locked", pp);
}
if (pp->p_selock == 0) {
/*
* If the T_CAPTURING bit is set, that means that we should
* not try and capture the page again as we could recurse
* which could lead to a stack overflow panic or spending a
* relatively long time in the kernel making no progress.
*/
if ((pp->p_toxic & PR_CAPTURE) &&
!(curthread->t_flag & T_CAPTURING) &&
!PP_RETIRED(pp)) {
THREAD_KPRI_REQUEST();
pp->p_selock = SE_WRITER;
mutex_exit(pse);
page_unlock_capture(pp);
} else {
mutex_exit(pse);
}
} else {
mutex_exit(pse);
}
}
/*
* Try to upgrade the lock on the page from a "shared" to an
* "exclusive" lock. Since this upgrade operation is done while
* holding the mutex protecting this page, no one else can acquire this page's
* lock and change the page. Thus, it is safe to drop the "shared"
* lock and attempt to acquire the "exclusive" lock.
*
* Returns 1 on success, 0 on failure.
*/
int
page_tryupgrade(page_t *pp)
{
kmutex_t *pse = PAGE_SE_MUTEX(pp);
mutex_enter(pse);
if (!(pp->p_selock & SE_EWANTED)) {
/* no threads want exclusive access, try upgrade */
if (pp->p_selock == SE_READER) {
THREAD_KPRI_REQUEST();
/* convert to exclusive lock */
pp->p_selock = SE_WRITER;
mutex_exit(pse);
return (1);
}
}
mutex_exit(pse);
return (0);
}
/*ARGSUSED*/
int64_t
loadable_syscall(
long a0, long a1, long a2, long a3,
long a4, long a5, long a6, long a7)
{
int64_t rval;
struct sysent *callp;
struct sysent *se = LWP_GETSYSENT(ttolwp(curthread));
krwlock_t *module_lock;
int code;
code = curthread->t_sysnum;
callp = se + code;
/*
* Try to autoload the system call if necessary.
*/
module_lock = lock_syscall(se, code);
THREAD_KPRI_RELEASE(); /* drop priority given by rw_enter */
/*
* we've locked either the loaded syscall or nosys
*/
if (callp->sy_flags & SE_ARGC) {
int64_t (*sy_call)();
sy_call = (int64_t (*)())callp->sy_call;
rval = (*sy_call)(a0, a1, a2, a3, a4, a5);
} else {
rval = syscall_ap();
}
THREAD_KPRI_REQUEST(); /* regain priority from read lock */
rw_exit(module_lock);
return (rval);
}
/*
* Read the comments inside of page_lock_es() carefully.
*
* SE_EXCL callers specifying es == SE_EXCL_WANTED will cause the
* SE_EWANTED bit of p_selock to be set when the lock cannot be obtained.
* This is used by threads subject to reader-starvation (eg. memory delete).
*
* When a thread using SE_EXCL_WANTED does not obtain the SE_EXCL lock,
* it is expected that it will retry at a later time. Threads that will
* not retry the lock *must* call page_lock_clr_exclwanted to clear the
* SE_EWANTED bit. (When a thread using SE_EXCL_WANTED obtains the lock,
* the bit is cleared.)
*/
int
page_try_reclaim_lock(page_t *pp, se_t se, int es)
{
kmutex_t *pse = PAGE_SE_MUTEX(pp);
selock_t old;
mutex_enter(pse);
old = pp->p_selock;
ASSERT(((es & SE_EXCL_WANTED) == 0) ||
((es & SE_EXCL_WANTED) && (se == SE_EXCL)));
if (PP_RETIRED(pp) && !(es & SE_RETIRED)) {
mutex_exit(pse);
VM_STAT_ADD(page_trylock_failed);
return (0);
}
if (se == SE_SHARED && es == 1 && old == 0) {
se = SE_EXCL;
}
if (se == SE_SHARED) {
if (!PP_ISFREE(pp)) {
if (old >= 0) {
/*
* Readers are not allowed when excl wanted
*/
if ((old & SE_EWANTED) == 0) {
pp->p_selock = old + SE_READER;
mutex_exit(pse);
return (1);
}
}
mutex_exit(pse);
return (0);
}
/*
* The page is free, so we really want SE_EXCL (below)
*/
VM_STAT_ADD(page_try_reclaim_upgrade);
}
/*
* The caller wants a writer lock. We try for it only if
* SE_EWANTED is not set, or if the caller specified
* SE_EXCL_WANTED.
*/
if (!(old & SE_EWANTED) || (es & SE_EXCL_WANTED)) {
if ((old & ~SE_EWANTED) == 0) {
/* no reader/writer lock held */
THREAD_KPRI_REQUEST();
/* this clears out our setting of the SE_EWANTED bit */
pp->p_selock = SE_WRITER;
mutex_exit(pse);
return (1);
}
}
if (es & SE_EXCL_WANTED) {
/* page is locked, set the SE_EWANTED bit */
pp->p_selock |= SE_EWANTED;
}
mutex_exit(pse);
return (0);
}
/*
* With the addition of reader-writer lock semantics to page_lock_es,
* callers wanting an exclusive (writer) lock may prevent shared-lock
* (reader) starvation by setting the es parameter to SE_EXCL_WANTED.
* In this case, when an exclusive lock cannot be acquired, p_selock's
* SE_EWANTED bit is set. Shared-lock (reader) requests are also denied
* if the page is slated for retirement.
*
* The se and es parameters determine if the lock should be granted
* based on the following decision table:
*
* Lock wanted es flags p_selock/SE_EWANTED Action
* ----------- -------------- ------------------- ---------
* SE_EXCL any [1][2] unlocked/any grant lock, clear SE_EWANTED
* SE_EXCL SE_EWANTED any lock/any deny, set SE_EWANTED
* SE_EXCL none any lock/any deny
* SE_SHARED n/a [2] shared/0 grant
* SE_SHARED n/a [2] unlocked/0 grant
* SE_SHARED n/a shared/1 deny
* SE_SHARED n/a unlocked/1 deny
* SE_SHARED n/a excl/any deny
*
* Notes:
* [1] The code grants an exclusive lock to the caller and clears the bit
* SE_EWANTED whenever p_selock is unlocked, regardless of the SE_EWANTED
* bit's value. This was deemed acceptable as we are not concerned about
* exclusive-lock starvation. If this ever becomes an issue, a priority or
* fifo mechanism should also be implemented. Meantime, the thread that
* set SE_EWANTED should be prepared to catch this condition and reset it
*
* [2] Retired pages may not be locked at any time, regardless of the
* dispostion of se, unless the es parameter has SE_RETIRED flag set.
*
* Notes on values of "es":
*
* es & 1: page_lookup_create will attempt page relocation
* es & SE_EXCL_WANTED: caller wants SE_EWANTED set (eg. delete
* memory thread); this prevents reader-starvation of waiting
* writer thread(s) by giving priority to writers over readers.
* es & SE_RETIRED: caller wants to lock pages even if they are
* retired. Default is to deny the lock if the page is retired.
*
* And yes, we know, the semantics of this function are too complicated.
* It's on the list to be cleaned up.
*/
int
page_lock_es(page_t *pp, se_t se, kmutex_t *lock, reclaim_t reclaim, int es)
{
int retval;
kmutex_t *pse = PAGE_SE_MUTEX(pp);
int upgraded;
int reclaim_it;
ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
VM_STAT_ADD(page_lock_count);
upgraded = 0;
reclaim_it = 0;
mutex_enter(pse);
ASSERT(((es & SE_EXCL_WANTED) == 0) ||
((es & SE_EXCL_WANTED) && (se == SE_EXCL)));
if (PP_RETIRED(pp) && !(es & SE_RETIRED)) {
mutex_exit(pse);
VM_STAT_ADD(page_lock_retired);
return (0);
}
if (se == SE_SHARED && es == 1 && pp->p_selock == 0) {
se = SE_EXCL;
}
if ((reclaim == P_RECLAIM) && (PP_ISFREE(pp))) {
reclaim_it = 1;
if (se == SE_SHARED) {
/*
* This is an interesting situation.
*
* Remember that p_free can only change if
* p_selock < 0.
* p_free does not depend on our holding `pse'.
* And, since we hold `pse', p_selock can not change.
* So, if p_free changes on us, the page is already
* exclusively held, and we would fail to get p_selock
* regardless.
*
* We want to avoid getting the share
* lock on a free page that needs to be reclaimed.
* It is possible that some other thread has the share
* lock and has left the free page on the cache list.
* pvn_vplist_dirty() does this for brief periods.
* If the se_share is currently SE_EXCL, we will fail
* to acquire p_selock anyway. Blocking is the
* right thing to do.
* If we need to reclaim this page, we must get
* exclusive access to it, force the upgrade now.
* Again, we will fail to acquire p_selock if the
* page is not free and block.
*/
upgraded = 1;
se = SE_EXCL;
VM_STAT_ADD(page_lock_upgrade);
}
}
if (se == SE_EXCL) {
if (!(es & SE_EXCL_WANTED) && (pp->p_selock & SE_EWANTED)) {
/*
* if the caller wants a writer lock (but did not
* specify exclusive access), and there is a pending
* writer that wants exclusive access, return failure
*/
retval = 0;
} else if ((pp->p_selock & ~SE_EWANTED) == 0) {
/* no reader/writer lock held */
THREAD_KPRI_REQUEST();
/* this clears our setting of the SE_EWANTED bit */
pp->p_selock = SE_WRITER;
retval = 1;
} else {
/* page is locked */
if (es & SE_EXCL_WANTED) {
/* set the SE_EWANTED bit */
pp->p_selock |= SE_EWANTED;
}
retval = 0;
}
} else {
retval = 0;
if (pp->p_selock >= 0) {
if ((pp->p_selock & SE_EWANTED) == 0) {
pp->p_selock += SE_READER;
retval = 1;
}
}
}
if (retval == 0) {
if ((pp->p_selock & ~SE_EWANTED) == SE_DELETED) {
VM_STAT_ADD(page_lock_deleted);
mutex_exit(pse);
return (retval);
}
#ifdef VM_STATS
VM_STAT_ADD(page_lock_miss);
if (upgraded) {
VM_STAT_ADD(page_lock_upgrade_failed);
}
#endif
if (lock) {
VM_STAT_ADD(page_lock_miss_lock);
mutex_exit(lock);
}
/*
* Now, wait for the page to be unlocked and
* release the lock protecting p_cv and p_selock.
*/
cv_wait(&pp->p_cv, pse);
mutex_exit(pse);
/*
* The page identity may have changed while we were
* blocked. If we are willing to depend on "pp"
* still pointing to a valid page structure (i.e.,
* assuming page structures are not dynamically allocated
* or freed), we could try to lock the page if its
* identity hasn't changed.
*
* This needs to be measured, since we come back from
* cv_wait holding pse (the expensive part of this
* operation) we might as well try the cheap part.
* Though we would also have to confirm that dropping
* `lock' did not cause any grief to the callers.
*/
if (lock) {
mutex_enter(lock);
}
} else {
/*
* We have the page lock.
* If we needed to reclaim the page, and the page
* needed reclaiming (ie, it was free), then we
* have the page exclusively locked. We may need
* to downgrade the page.
*/
ASSERT((upgraded) ?
((PP_ISFREE(pp)) && PAGE_EXCL(pp)) : 1);
mutex_exit(pse);
/*
* We now hold this page's lock, either shared or
* exclusive. This will prevent its identity from changing.
* The page, however, may or may not be free. If the caller
* requested, and it is free, go reclaim it from the
* free list. If the page can't be reclaimed, return failure
* so that the caller can start all over again.
*
* NOTE:page_reclaim() releases the page lock (p_selock)
* if it can't be reclaimed.
*/
if (reclaim_it) {
if (!page_reclaim(pp, lock)) {
VM_STAT_ADD(page_lock_bad_reclaim);
retval = 0;
} else {
VM_STAT_ADD(page_lock_reclaim);
if (upgraded) {
page_downgrade(pp);
}
}
}
}
return (retval);
}
