xref: /netvirt/usr/src/uts/common/syscall/lwp_sobj.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved	*/

#pragma ident	"@(#)lwp_sobj.c	1.77	07/06/17 SMI"

#include <sys/param.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/cred.h>
#include <sys/user.h>
#include <sys/errno.h>
#include <sys/file.h>
#include <sys/proc.h>
#include <sys/prsystm.h>
#include <sys/kmem.h>
#include <sys/sobject.h>
#include <sys/fault.h>
#include <sys/procfs.h>
#include <sys/watchpoint.h>
#include <sys/time.h>
#include <sys/cmn_err.h>
#include <sys/machlock.h>
#include <sys/debug.h>
#include <sys/synch.h>
#include <sys/synch32.h>
#include <sys/mman.h>
#include <sys/class.h>
#include <sys/schedctl.h>
#include <sys/sleepq.h>
#include <sys/policy.h>
#include <sys/tnf_probe.h>
#include <sys/lwpchan_impl.h>
#include <sys/turnstile.h>
#include <sys/atomic.h>
#include <sys/lwp_timer_impl.h>
#include <sys/lwp_upimutex_impl.h>
#include <vm/as.h>
#include <sys/sdt.h>

static kthread_t *lwpsobj_owner(caddr_t);
static void lwp_unsleep(kthread_t *t);
static void lwp_change_pri(kthread_t *t, pri_t pri, pri_t *t_prip);
static void lwp_mutex_cleanup(lwpchan_entry_t *ent, uint16_t lockflg);

extern int lwp_cond_signal(lwp_cond_t *cv);

/*
 * Maximum number of user prio inheritance locks that can be held by a thread.
 * Used to limit kmem for each thread. This is a per-thread limit that
 * can be administered on a system wide basis (using /etc/system).
 *
 * Also, when a limit, say maxlwps is added for numbers of lwps within a
 * process, the per-thread limit automatically becomes a process-wide limit
 * of maximum number of held upi locks within a process:
 *      maxheldupimx = maxnestupimx * maxlwps;
 */
static uint32_t maxnestupimx = 2000;

/*
 * The sobj_ops vector exports a set of functions needed when a thread
 * is asleep on a synchronization object of this type.
 */
static sobj_ops_t lwp_sobj_ops = {
	SOBJ_USER, lwpsobj_owner, lwp_unsleep, lwp_change_pri
};

static kthread_t *lwpsobj_pi_owner(upimutex_t *up);

static sobj_ops_t lwp_sobj_pi_ops = {
	SOBJ_USER_PI, lwpsobj_pi_owner, turnstile_unsleep,
	turnstile_change_pri
};

static sleepq_head_t	lwpsleepq[NSLEEPQ];
upib_t			upimutextab[UPIMUTEX_TABSIZE];

#define	LWPCHAN_LOCK_SHIFT	10	/* 1024 locks for each pool */
#define	LWPCHAN_LOCK_SIZE	(1 << LWPCHAN_LOCK_SHIFT)

/*
 * We know that both lc_wchan and lc_wchan0 are addresses that most
 * likely are 8-byte aligned, so we shift off the low-order 3 bits.
 * 'pool' is either 0 or 1.
 */
#define	LWPCHAN_LOCK_HASH(X, pool) \
	(((((X) >> 3) ^ ((X) >> (LWPCHAN_LOCK_SHIFT + 3))) & \
	(LWPCHAN_LOCK_SIZE - 1)) + ((pool)? LWPCHAN_LOCK_SIZE : 0))

static kmutex_t		lwpchanlock[2 * LWPCHAN_LOCK_SIZE];

/*
 * Is this a POSIX threads user-level lock requiring priority inheritance?
 */
#define	UPIMUTEX(type)	((type) & LOCK_PRIO_INHERIT)

static sleepq_head_t *
lwpsqhash(lwpchan_t *lwpchan)
{
	uint_t x = (uintptr_t)lwpchan->lc_wchan ^ (uintptr_t)lwpchan->lc_wchan0;
	return (&lwpsleepq[SQHASHINDEX(x)]);
}

/*
 * Lock an lwpchan.
 * Keep this in sync with lwpchan_unlock(), below.
 */
static void
lwpchan_lock(lwpchan_t *lwpchan, int pool)
{
	uint_t x = (uintptr_t)lwpchan->lc_wchan ^ (uintptr_t)lwpchan->lc_wchan0;
	mutex_enter(&lwpchanlock[LWPCHAN_LOCK_HASH(x, pool)]);
}

/*
 * Unlock an lwpchan.
 * Keep this in sync with lwpchan_lock(), above.
 */
static void
lwpchan_unlock(lwpchan_t *lwpchan, int pool)
{
	uint_t x = (uintptr_t)lwpchan->lc_wchan ^ (uintptr_t)lwpchan->lc_wchan0;
	mutex_exit(&lwpchanlock[LWPCHAN_LOCK_HASH(x, pool)]);
}

/*
 * Delete mappings from the lwpchan cache for pages that are being
 * unmapped by as_unmap().  Given a range of addresses, "start" to "end",
 * all mappings within the range are deleted from the lwpchan cache.
 */
void
lwpchan_delete_mapping(proc_t *p, caddr_t start, caddr_t end)
{
	lwpchan_data_t *lcp;
	lwpchan_hashbucket_t *hashbucket;
	lwpchan_hashbucket_t *endbucket;
	lwpchan_entry_t *ent;
	lwpchan_entry_t **prev;
	caddr_t addr;

	mutex_enter(&p->p_lcp_lock);
	lcp = p->p_lcp;
	hashbucket = lcp->lwpchan_cache;
	endbucket = hashbucket + lcp->lwpchan_size;
	for (; hashbucket < endbucket; hashbucket++) {
		if (hashbucket->lwpchan_chain == NULL)
			continue;
		mutex_enter(&hashbucket->lwpchan_lock);
		prev = &hashbucket->lwpchan_chain;
		/* check entire chain */
		while ((ent = *prev) != NULL) {
			addr = ent->lwpchan_addr;
			if (start <= addr && addr < end) {
				*prev = ent->lwpchan_next;
				if (ent->lwpchan_pool == LWPCHAN_MPPOOL &&
				    (ent->lwpchan_type & LOCK_ROBUST))
					lwp_mutex_cleanup(ent, LOCK_UNMAPPED);
				kmem_free(ent, sizeof (*ent));
				atomic_add_32(&lcp->lwpchan_entries, -1);
			} else {
				prev = &ent->lwpchan_next;
			}
		}
		mutex_exit(&hashbucket->lwpchan_lock);
	}
	mutex_exit(&p->p_lcp_lock);
}

/*
 * Given an lwpchan cache pointer and a process virtual address,
 * return a pointer to the corresponding lwpchan hash bucket.
 */
static lwpchan_hashbucket_t *
lwpchan_bucket(lwpchan_data_t *lcp, uintptr_t addr)
{
	uint_t i;

	/*
	 * All user-level sync object addresses are 8-byte aligned.
	 * Ignore the lowest 3 bits of the address and use the
	 * higher-order 2*lwpchan_bits bits for the hash index.
	 */
	addr >>= 3;
	i = (addr ^ (addr >> lcp->lwpchan_bits)) & lcp->lwpchan_mask;
	return (lcp->lwpchan_cache + i);
}

/*
 * (Re)allocate the per-process lwpchan cache.
 */
static void
lwpchan_alloc_cache(proc_t *p, uint_t bits)
{
	lwpchan_data_t *lcp;
	lwpchan_data_t *old_lcp;
	lwpchan_hashbucket_t *hashbucket;
	lwpchan_hashbucket_t *endbucket;
	lwpchan_hashbucket_t *newbucket;
	lwpchan_entry_t *ent;
	lwpchan_entry_t *next;
	uint_t count;

	ASSERT(bits >= LWPCHAN_INITIAL_BITS && bits <= LWPCHAN_MAX_BITS);

	lcp = kmem_alloc(sizeof (lwpchan_data_t), KM_SLEEP);
	lcp->lwpchan_bits = bits;
	lcp->lwpchan_size = 1 << lcp->lwpchan_bits;
	lcp->lwpchan_mask = lcp->lwpchan_size - 1;
	lcp->lwpchan_entries = 0;
	lcp->lwpchan_cache = kmem_zalloc(lcp->lwpchan_size *
		sizeof (lwpchan_hashbucket_t), KM_SLEEP);
	lcp->lwpchan_next_data = NULL;

	mutex_enter(&p->p_lcp_lock);
	if ((old_lcp = p->p_lcp) != NULL) {
		if (old_lcp->lwpchan_bits >= bits) {
			/* someone beat us to it */
			mutex_exit(&p->p_lcp_lock);
			kmem_free(lcp->lwpchan_cache, lcp->lwpchan_size *
				sizeof (lwpchan_hashbucket_t));
			kmem_free(lcp, sizeof (lwpchan_data_t));
			return;
		}
		/*
		 * Acquire all of the old hash table locks.
		 */
		hashbucket = old_lcp->lwpchan_cache;
		endbucket = hashbucket + old_lcp->lwpchan_size;
		for (; hashbucket < endbucket; hashbucket++)
			mutex_enter(&hashbucket->lwpchan_lock);
		/*
		 * Move all of the old hash table entries to the
		 * new hash table.  The new hash table has not yet
		 * been installed so we don't need any of its locks.
		 */
		count = 0;
		hashbucket = old_lcp->lwpchan_cache;
		for (; hashbucket < endbucket; hashbucket++) {
			ent = hashbucket->lwpchan_chain;
			while (ent != NULL) {
				next = ent->lwpchan_next;
				newbucket = lwpchan_bucket(lcp,
					(uintptr_t)ent->lwpchan_addr);
				ent->lwpchan_next = newbucket->lwpchan_chain;
				newbucket->lwpchan_chain = ent;
				ent = next;
				count++;
			}
			hashbucket->lwpchan_chain = NULL;
		}
		lcp->lwpchan_entries = count;
	}

	/*
	 * Retire the old hash table.  We can't actually kmem_free() it
	 * now because someone may still have a pointer to it.  Instead,
	 * we link it onto the new hash table's list of retired hash tables.
	 * The new hash table is double the size of the previous one, so
	 * the total size of all retired hash tables is less than the size
	 * of the new one.  exit() and exec() free the retired hash tables
	 * (see lwpchan_destroy_cache(), below).
	 */
	lcp->lwpchan_next_data = old_lcp;

	/*
	 * As soon as we store the new lcp, future locking operations will
	 * use it.  Therefore, we must ensure that all the state we've just
	 * established reaches global visibility before the new lcp does.
	 */
	membar_producer();
	p->p_lcp = lcp;

	if (old_lcp != NULL) {
		/*
		 * Release all of the old hash table locks.
		 */
		hashbucket = old_lcp->lwpchan_cache;
		for (; hashbucket < endbucket; hashbucket++)
			mutex_exit(&hashbucket->lwpchan_lock);
	}
	mutex_exit(&p->p_lcp_lock);
}

/*
 * Deallocate the lwpchan cache, and any dynamically allocated mappings.
 * Called when the process exits or execs.  All lwps except one have
 * exited so we need no locks here.
 */
void
lwpchan_destroy_cache(int exec)
{
	proc_t *p = curproc;
	lwpchan_hashbucket_t *hashbucket;
	lwpchan_hashbucket_t *endbucket;
	lwpchan_data_t *lcp;
	lwpchan_entry_t *ent;
	lwpchan_entry_t *next;
	uint16_t lockflg;

	lcp = p->p_lcp;
	p->p_lcp = NULL;

	lockflg = exec? LOCK_UNMAPPED : LOCK_OWNERDEAD;
	hashbucket = lcp->lwpchan_cache;
	endbucket = hashbucket + lcp->lwpchan_size;
	for (; hashbucket < endbucket; hashbucket++) {
		ent = hashbucket->lwpchan_chain;
		hashbucket->lwpchan_chain = NULL;
		while (ent != NULL) {
			next = ent->lwpchan_next;
			if (ent->lwpchan_pool == LWPCHAN_MPPOOL &&
			    (ent->lwpchan_type & LOCK_ROBUST))
				lwp_mutex_cleanup(ent, lockflg);
			kmem_free(ent, sizeof (*ent));
			ent = next;
		}
	}

	while (lcp != NULL) {
		lwpchan_data_t *next_lcp = lcp->lwpchan_next_data;
		kmem_free(lcp->lwpchan_cache, lcp->lwpchan_size *
			sizeof (lwpchan_hashbucket_t));
		kmem_free(lcp, sizeof (lwpchan_data_t));
		lcp = next_lcp;
	}
}

/*
 * Return zero when there is an entry in the lwpchan cache for the
 * given process virtual address and non-zero when there is not.
 * The returned non-zero value is the current length of the
 * hash chain plus one.  The caller holds the hash bucket lock.
 */
static uint_t
lwpchan_cache_mapping(caddr_t addr, int type, int pool, lwpchan_t *lwpchan,
	lwpchan_hashbucket_t *hashbucket)
{
	lwpchan_entry_t *ent;
	uint_t count = 1;

	for (ent = hashbucket->lwpchan_chain; ent; ent = ent->lwpchan_next) {
		if (ent->lwpchan_addr == addr) {
			if (ent->lwpchan_type != type ||
			    ent->lwpchan_pool != pool) {
				/*
				 * This shouldn't happen, but might if the
				 * process reuses its memory for different
				 * types of sync objects.  We test first
				 * to avoid grabbing the memory cache line.
				 */
				ent->lwpchan_type = (uint16_t)type;
				ent->lwpchan_pool = (uint16_t)pool;
			}
			*lwpchan = ent->lwpchan_lwpchan;
			return (0);
		}
		count++;
	}
	return (count);
}

/*
 * Return the cached lwpchan mapping if cached, otherwise insert
 * a virtual address to lwpchan mapping into the cache.
 */
static int
lwpchan_get_mapping(struct as *as, caddr_t addr,
	int type, lwpchan_t *lwpchan, int pool)
{
	proc_t *p = curproc;
	lwpchan_data_t *lcp;
	lwpchan_hashbucket_t *hashbucket;
	lwpchan_entry_t *ent;
	memid_t	memid;
	uint_t count;
	uint_t bits;

top:
	/* initialize the lwpchan cache, if necesary */
	if ((lcp = p->p_lcp) == NULL) {
		lwpchan_alloc_cache(p, LWPCHAN_INITIAL_BITS);
		goto top;
	}
	hashbucket = lwpchan_bucket(lcp, (uintptr_t)addr);
	mutex_enter(&hashbucket->lwpchan_lock);
	if (lcp != p->p_lcp) {
		/* someone resized the lwpchan cache; start over */
		mutex_exit(&hashbucket->lwpchan_lock);
		goto top;
	}
	if (lwpchan_cache_mapping(addr, type, pool, lwpchan, hashbucket) == 0) {
		/* it's in the cache */
		mutex_exit(&hashbucket->lwpchan_lock);
		return (1);
	}
	mutex_exit(&hashbucket->lwpchan_lock);
	if (as_getmemid(as, addr, &memid) != 0)
		return (0);
	lwpchan->lc_wchan0 = (caddr_t)(uintptr_t)memid.val[0];
	lwpchan->lc_wchan = (caddr_t)(uintptr_t)memid.val[1];
	ent = kmem_alloc(sizeof (lwpchan_entry_t), KM_SLEEP);
	mutex_enter(&hashbucket->lwpchan_lock);
	if (lcp != p->p_lcp) {
		/* someone resized the lwpchan cache; start over */
		mutex_exit(&hashbucket->lwpchan_lock);
		kmem_free(ent, sizeof (*ent));
		goto top;
	}
	count = lwpchan_cache_mapping(addr, type, pool, lwpchan, hashbucket);
	if (count == 0) {
		/* someone else added this entry to the cache */
		mutex_exit(&hashbucket->lwpchan_lock);
		kmem_free(ent, sizeof (*ent));
		return (1);
	}
	if (count > lcp->lwpchan_bits + 2 && /* larger table, longer chains */
	    (bits = lcp->lwpchan_bits) < LWPCHAN_MAX_BITS) {
		/* hash chain too long; reallocate the hash table */
		mutex_exit(&hashbucket->lwpchan_lock);
		kmem_free(ent, sizeof (*ent));
		lwpchan_alloc_cache(p, bits + 1);
		goto top;
	}
	ent->lwpchan_addr = addr;
	ent->lwpchan_type = (uint16_t)type;
	ent->lwpchan_pool = (uint16_t)pool;
	ent->lwpchan_lwpchan = *lwpchan;
	ent->lwpchan_next = hashbucket->lwpchan_chain;
	hashbucket->lwpchan_chain = ent;
	atomic_add_32(&lcp->lwpchan_entries, 1);
	mutex_exit(&hashbucket->lwpchan_lock);
	return (1);
}

/*
 * Return a unique pair of identifiers that corresponds to a
 * synchronization object's virtual address.  Process-shared
 * sync objects usually get vnode/offset from as_getmemid().
 */
static int
get_lwpchan(struct as *as, caddr_t addr, int type, lwpchan_t *lwpchan, int pool)
{
	/*
	 * If the lwp synch object is defined to be process-private,
	 * we just make the first field of the lwpchan be 'as' and
	 * the second field be the synch object's virtual address.
	 * (segvn_getmemid() does the same for MAP_PRIVATE mappings.)
	 * The lwpchan cache is used only for process-shared objects.
	 */
	if (!(type & USYNC_PROCESS)) {
		lwpchan->lc_wchan0 = (caddr_t)as;
		lwpchan->lc_wchan = addr;
		return (1);
	}

	return (lwpchan_get_mapping(as, addr, type, lwpchan, pool));
}

static void
lwp_block(lwpchan_t *lwpchan)
{
	kthread_t *t = curthread;
	klwp_t *lwp = ttolwp(t);
	sleepq_head_t *sqh;

	thread_lock(t);
	t->t_flag |= T_WAKEABLE;
	t->t_lwpchan = *lwpchan;
	t->t_sobj_ops = &lwp_sobj_ops;
	t->t_release = 0;
	sqh = lwpsqhash(lwpchan);
	disp_lock_enter_high(&sqh->sq_lock);
	CL_SLEEP(t);
	DTRACE_SCHED(sleep);
	THREAD_SLEEP(t, &sqh->sq_lock);
	sleepq_insert(&sqh->sq_queue, t);
	thread_unlock(t);
	lwp->lwp_asleep = 1;
	lwp->lwp_sysabort = 0;
	lwp->lwp_ru.nvcsw++;
	(void) new_mstate(curthread, LMS_SLEEP);
}

static kthread_t *
lwpsobj_pi_owner(upimutex_t *up)
{
	return (up->upi_owner);
}

static struct upimutex *
upi_get(upib_t *upibp, lwpchan_t *lcp)
{
	struct upimutex *upip;

	for (upip = upibp->upib_first; upip != NULL;
	    upip = upip->upi_nextchain) {
		if (upip->upi_lwpchan.lc_wchan0 == lcp->lc_wchan0 &&
		    upip->upi_lwpchan.lc_wchan == lcp->lc_wchan)
			break;
	}
	return (upip);
}

static void
upi_chain_add(upib_t *upibp, struct upimutex *upimutex)
{
	ASSERT(MUTEX_HELD(&upibp->upib_lock));

	/*
	 * Insert upimutex at front of list. Maybe a bit unfair
	 * but assume that not many lwpchans hash to the same
	 * upimutextab bucket, i.e. the list of upimutexes from
	 * upib_first is not too long.
	 */
	upimutex->upi_nextchain = upibp->upib_first;
	upibp->upib_first = upimutex;
}

static void
upi_chain_del(upib_t *upibp, struct upimutex *upimutex)
{
	struct upimutex **prev;

	ASSERT(MUTEX_HELD(&upibp->upib_lock));

	prev = &upibp->upib_first;
	while (*prev != upimutex) {
		prev = &(*prev)->upi_nextchain;
	}
	*prev = upimutex->upi_nextchain;
	upimutex->upi_nextchain = NULL;
}

/*
 * Add upimutex to chain of upimutexes held by curthread.
 * Returns number of upimutexes held by curthread.
 */
static uint32_t
upi_mylist_add(struct upimutex *upimutex)
{
	kthread_t *t = curthread;

	/*
	 * Insert upimutex at front of list of upimutexes owned by t. This
	 * would match typical LIFO order in which nested locks are acquired
	 * and released.
	 */
	upimutex->upi_nextowned = t->t_upimutex;
	t->t_upimutex = upimutex;
	t->t_nupinest++;
	ASSERT(t->t_nupinest > 0);
	return (t->t_nupinest);
}

/*
 * Delete upimutex from list of upimutexes owned by curthread.
 */
static void
upi_mylist_del(struct upimutex *upimutex)
{
	kthread_t *t = curthread;
	struct upimutex **prev;

	/*
	 * Since the order in which nested locks are acquired and released,
	 * is typically LIFO, and typical nesting levels are not too deep, the
	 * following should not be expensive in the general case.
	 */
	prev = &t->t_upimutex;
	while (*prev != upimutex) {
		prev = &(*prev)->upi_nextowned;
	}
	*prev = upimutex->upi_nextowned;
	upimutex->upi_nextowned = NULL;
	ASSERT(t->t_nupinest > 0);
	t->t_nupinest--;
}

/*
 * Returns true if upimutex is owned. Should be called only when upim points
 * to kmem which cannot disappear from underneath.
 */
static int
upi_owned(upimutex_t *upim)
{
	return (upim->upi_owner == curthread);
}

/*
 * Returns pointer to kernel object (upimutex_t *) if lp is owned.
 */
static struct upimutex *
lwp_upimutex_owned(lwp_mutex_t *lp, uint8_t type)
{
	lwpchan_t lwpchan;
	upib_t *upibp;
	struct upimutex *upimutex;

	if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
	    &lwpchan, LWPCHAN_MPPOOL))
		return (NULL);

	upibp = &UPI_CHAIN(lwpchan);
	mutex_enter(&upibp->upib_lock);
	upimutex = upi_get(upibp, &lwpchan);
	if (upimutex == NULL || upimutex->upi_owner != curthread) {
		mutex_exit(&upibp->upib_lock);
		return (NULL);
	}
	mutex_exit(&upibp->upib_lock);
	return (upimutex);
}

/*
 * Unlocks upimutex, waking up waiters if any. upimutex kmem is freed if
 * no lock hand-off occurrs.
 */
static void
upimutex_unlock(struct upimutex *upimutex, uint16_t flag)
{
	turnstile_t *ts;
	upib_t *upibp;
	kthread_t *newowner;

	upi_mylist_del(upimutex);
	upibp = upimutex->upi_upibp;
	mutex_enter(&upibp->upib_lock);
	if (upimutex->upi_waiter != 0) { /* if waiters */
		ts = turnstile_lookup(upimutex);
		if (ts != NULL && !(flag & LOCK_NOTRECOVERABLE)) {
			/* hand-off lock to highest prio waiter */
			newowner = ts->ts_sleepq[TS_WRITER_Q].sq_first;
			upimutex->upi_owner = newowner;
			if (ts->ts_waiters == 1)
				upimutex->upi_waiter = 0;
			turnstile_wakeup(ts, TS_WRITER_Q, 1, newowner);
			mutex_exit(&upibp->upib_lock);
			return;
		} else if (ts != NULL) {
			/* LOCK_NOTRECOVERABLE: wakeup all */
			turnstile_wakeup(ts, TS_WRITER_Q, ts->ts_waiters, NULL);
		} else {
			/*
			 * Misleading w bit. Waiters might have been
			 * interrupted. No need to clear the w bit (upimutex
			 * will soon be freed). Re-calculate PI from existing
			 * waiters.
			 */
			turnstile_exit(upimutex);
			turnstile_pi_recalc();
		}
	}
	/*
	 * no waiters, or LOCK_NOTRECOVERABLE.
	 * remove from the bucket chain of upi mutexes.
	 * de-allocate kernel memory (upimutex).
	 */
	upi_chain_del(upimutex->upi_upibp, upimutex);
	mutex_exit(&upibp->upib_lock);
	kmem_free(upimutex, sizeof (upimutex_t));
}

static int
lwp_upimutex_lock(lwp_mutex_t *lp, uint8_t type, int try, lwp_timer_t *lwptp)
{
	label_t ljb;
	int error = 0;
	lwpchan_t lwpchan;
	uint16_t flag;
	upib_t *upibp;
	volatile struct upimutex *upimutex = NULL;
	turnstile_t *ts;
	uint32_t nupinest;
	volatile int upilocked = 0;

	if (on_fault(&ljb)) {
		if (upilocked)
			upimutex_unlock((upimutex_t *)upimutex, 0);
		error = EFAULT;
		goto out;
	}
	/*
	 * The apparent assumption made in implementing other _lwp_* synch
	 * primitives, is that get_lwpchan() does not return a unique cookie
	 * for the case where 2 processes (one forked from the other) point
	 * at the same underlying object, which is typed USYNC_PROCESS, but
	 * mapped MAP_PRIVATE, since the object has not yet been written to,
	 * in the child process.
	 *
	 * Since get_lwpchan() has been fixed, it is not necessary to do the
	 * dummy writes to force a COW fault as in other places (which should
	 * be fixed).
	 */
	if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
	    &lwpchan, LWPCHAN_MPPOOL)) {
		error = EFAULT;
		goto out;
	}
	upibp = &UPI_CHAIN(lwpchan);
retry:
	mutex_enter(&upibp->upib_lock);
	upimutex = upi_get(upibp, &lwpchan);
	if (upimutex == NULL)  {
		/* lock available since lwpchan has no upimutex */
		upimutex = kmem_zalloc(sizeof (upimutex_t), KM_SLEEP);
		upi_chain_add(upibp, (upimutex_t *)upimutex);
		upimutex->upi_owner = curthread; /* grab lock */
		upimutex->upi_upibp = upibp;
		upimutex->upi_vaddr = lp;
		upimutex->upi_lwpchan = lwpchan;
		mutex_exit(&upibp->upib_lock);
		nupinest = upi_mylist_add((upimutex_t *)upimutex);
		upilocked = 1;
		fuword16_noerr(&lp->mutex_flag, &flag);
		if (nupinest > maxnestupimx &&
		    secpolicy_resource(CRED()) != 0) {
			upimutex_unlock((upimutex_t *)upimutex, flag);
			error = ENOMEM;
			goto out;
		}
		if (flag & LOCK_NOTRECOVERABLE) {
			/*
			 * Since the setting of LOCK_NOTRECOVERABLE
			 * was done under the high-level upi mutex,
			 * in lwp_upimutex_unlock(), this flag needs to
			 * be checked while holding the upi mutex.
			 * If set, this thread should return without
			 * the lock held, and with the right error code.
			 */
			upimutex_unlock((upimutex_t *)upimutex, flag);
			upilocked = 0;
			error = ENOTRECOVERABLE;
		} else if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
			if (flag & LOCK_OWNERDEAD)
				error = EOWNERDEAD;
			else if (type & USYNC_PROCESS_ROBUST)
				error = ELOCKUNMAPPED;
			else
				error = EOWNERDEAD;
		}
		goto out;
	}
	/*
	 * If a upimutex object exists, it must have an owner.
	 * This is due to lock hand-off, and release of upimutex when no
	 * waiters are present at unlock time,
	 */
	ASSERT(upimutex->upi_owner != NULL);
	if (upimutex->upi_owner == curthread) {
		/*
		 * The user wrapper can check if the mutex type is
		 * ERRORCHECK: if not, it should stall at user-level.
		 * If so, it should return the error code.
		 */
		mutex_exit(&upibp->upib_lock);
		error = EDEADLK;
		goto out;
	}
	if (try == UPIMUTEX_TRY) {
		mutex_exit(&upibp->upib_lock);
		error = EBUSY;
		goto out;
	}
	/*
	 * Block for the lock.
	 * Put the lwp in an orderly state for debugging.
	 * Calling prstop() has to be done here, and not in
	 * turnstile_block(), since the preceding call to
	 * turnstile_lookup() raises the PIL to a level
	 * at which calls to prstop() should not be made.
	 */
	if ((error = lwptp->lwpt_time_error) != 0) {
		/*
		 * The SUSV3 Posix spec is very clear that we
		 * should get no error from validating the
		 * timer until we would actually sleep.
		 */
		mutex_exit(&upibp->upib_lock);
		goto out;
	}
	prstop(PR_REQUESTED, 0);
	if (lwptp->lwpt_tsp != NULL) {
		/*
		 * If we successfully queue the timeout
		 * (lwp_timer_enqueue() returns zero),
		 * then don't drop t_delay_lock until we are
		 * on the sleep queue (in turnstile_block()).
		 * Otherwise we will get an immediate timeout
		 * when we attempt to sleep in turnstile_block().
		 */
		mutex_enter(&curthread->t_delay_lock);
		if (lwp_timer_enqueue(lwptp) != 0)
			mutex_exit(&curthread->t_delay_lock);
	}
	/*
	 * Now, set the waiter bit and block for the lock in turnstile_block().
	 * No need to preserve the previous wbit since a lock try is not
	 * attempted after setting the wait bit. Wait bit is set under
	 * the upib_lock, which is not released until the turnstile lock
	 * is acquired. Say, the upimutex is L:
	 *
	 * 1. upib_lock is held so the waiter does not have to retry L after
	 *    setting the wait bit: since the owner has to grab the upib_lock
	 *    to unlock L, it will certainly see the wait bit set.
	 * 2. upib_lock is not released until the turnstile lock is acquired.
	 *    This is the key to preventing a missed wake-up. Otherwise, the
	 *    owner could acquire the upib_lock, and the tc_lock, to call
	 *    turnstile_wakeup(). All this, before the waiter gets tc_lock
	 *    to sleep in turnstile_block(). turnstile_wakeup() will then not
	 *    find this waiter, resulting in the missed wakeup.
	 * 3. The upib_lock, being a kernel mutex, cannot be released while
	 *    holding the tc_lock (since mutex_exit() could need to acquire
	 *    the same tc_lock)...and so is held when calling turnstile_block().
	 *    The address of upib_lock is passed to turnstile_block() which
	 *    releases it after releasing all turnstile locks, and before going
	 *    to sleep in swtch().
	 * 4. The waiter value cannot be a count of waiters, because a waiter
	 *    can be interrupted. The interrupt occurs under the tc_lock, at
	 *    which point, the upib_lock cannot be locked, to decrement waiter
	 *    count. So, just treat the waiter state as a bit, not a count.
	 */
	ts = turnstile_lookup((upimutex_t *)upimutex);
	upimutex->upi_waiter = 1;
	error = turnstile_block(ts, TS_WRITER_Q, (upimutex_t *)upimutex,
	    &lwp_sobj_pi_ops, &upibp->upib_lock, lwptp);
	/*
	 * Hand-off implies that we wakeup holding the lock, except when:
	 *	- deadlock is detected
	 *	- lock is not recoverable
	 *	- we got an interrupt or timeout
	 * If we wake up due to an interrupt or timeout, we may
	 * or may not be holding the lock due to mutex hand-off.
	 * Use lwp_upimutex_owned() to check if we do hold the lock.
	 */
	if (error != 0) {
		if ((error == EINTR || error == ETIME) &&
		    (upimutex = lwp_upimutex_owned(lp, type))) {
			/*
			 * Unlock and return - the re-startable syscall will
			 * try the lock again if we got EINTR.
			 */
			(void) upi_mylist_add((upimutex_t *)upimutex);
			upimutex_unlock((upimutex_t *)upimutex, 0);
		}
		/*
		 * The only other possible error is EDEADLK.  If so, upimutex
		 * is valid, since its owner is deadlocked with curthread.
		 */
		ASSERT(error == EINTR || error == ETIME ||
		    (error == EDEADLK && !upi_owned((upimutex_t *)upimutex)));
		ASSERT(!lwp_upimutex_owned(lp, type));
		goto out;
	}
	if (lwp_upimutex_owned(lp, type)) {
		ASSERT(lwp_upimutex_owned(lp, type) == upimutex);
		nupinest = upi_mylist_add((upimutex_t *)upimutex);
		upilocked = 1;
	}
	/*
	 * Now, need to read the user-level lp->mutex_flag to do the following:
	 *
	 * - if lock is held, check if EOWNERDEAD or ELOCKUNMAPPED
	 *   should be returned.
	 * - if lock isn't held, check if ENOTRECOVERABLE should
	 *   be returned.
	 *
	 * Now, either lp->mutex_flag is readable or it's not. If not
	 * readable, the on_fault path will cause a return with EFAULT
	 * as it should.  If it is readable, the state of the flag
	 * encodes the robustness state of the lock:
	 *
	 * If the upimutex is locked here, the flag's LOCK_OWNERDEAD
	 * or LOCK_UNMAPPED setting will influence the return code
	 * appropriately.  If the upimutex is not locked here, this
	 * could be due to a spurious wake-up or a NOTRECOVERABLE
	 * event.  The flag's setting can be used to distinguish
	 * between these two events.
	 */
	fuword16_noerr(&lp->mutex_flag, &flag);
	if (upilocked) {
		/*
		 * If the thread wakes up from turnstile_block with the lock
		 * held, the flag could not be set to LOCK_NOTRECOVERABLE,
		 * since it would not have been handed-off the lock.
		 * So, no need to check for this case.
		 */
		if (nupinest > maxnestupimx &&
		    secpolicy_resource(CRED()) != 0) {
			upimutex_unlock((upimutex_t *)upimutex, flag);
			upilocked = 0;
			error = ENOMEM;
		} else if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
			if (flag & LOCK_OWNERDEAD)
				error = EOWNERDEAD;
			else if (type & USYNC_PROCESS_ROBUST)
				error = ELOCKUNMAPPED;
			else
				error = EOWNERDEAD;
		}
	} else {
		/*
		 * Wake-up without the upimutex held. Either this is a
		 * spurious wake-up (due to signals, forkall(), whatever), or
		 * it is a LOCK_NOTRECOVERABLE robustness event. The setting
		 * of the mutex flag can be used to distinguish between the
		 * two events.
		 */
		if (flag & LOCK_NOTRECOVERABLE) {
			error = ENOTRECOVERABLE;
		} else {
			/*
			 * Here, the flag could be set to LOCK_OWNERDEAD or
			 * not. In both cases, this is a spurious wakeup,
			 * since the upi lock is not held, but the thread
			 * has returned from turnstile_block().
			 *
			 * The user flag could be LOCK_OWNERDEAD if, at the
			 * same time as curthread having been woken up
			 * spuriously, the owner (say Tdead) has died, marked
			 * the mutex flag accordingly, and handed off the lock
			 * to some other waiter (say Tnew). curthread just
			 * happened to read the flag while Tnew has yet to deal
			 * with the owner-dead event.
			 *
			 * In this event, curthread should retry the lock.
			 * If Tnew is able to cleanup the lock, curthread
			 * will eventually get the lock with a zero error code,
			 * If Tnew is unable to cleanup, its eventual call to
			 * unlock the lock will result in the mutex flag being
			 * set to LOCK_NOTRECOVERABLE, and the wake-up of
			 * all waiters, including curthread, which will then
			 * eventually return ENOTRECOVERABLE due to the above
			 * check.
			 *
			 * Of course, if the user-flag is not set with
			 * LOCK_OWNERDEAD, retrying is the thing to do, since
			 * this is definitely a spurious wakeup.
			 */
			goto retry;
		}
	}

out:
	no_fault();
	return (error);
}


static int
lwp_upimutex_unlock(lwp_mutex_t *lp, uint8_t type)
{
	label_t ljb;
	int error = 0;
	lwpchan_t lwpchan;
	uint16_t flag;
	upib_t *upibp;
	volatile struct upimutex *upimutex = NULL;
	volatile int upilocked = 0;

	if (on_fault(&ljb)) {
		if (upilocked)
			upimutex_unlock((upimutex_t *)upimutex, 0);
		error = EFAULT;
		goto out;
	}
	if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
	    &lwpchan, LWPCHAN_MPPOOL)) {
		error = EFAULT;
		goto out;
	}
	upibp = &UPI_CHAIN(lwpchan);
	mutex_enter(&upibp->upib_lock);
	upimutex = upi_get(upibp, &lwpchan);
	/*
	 * If the lock is not held, or the owner is not curthread, return
	 * error. The user-level wrapper can return this error or stall,
	 * depending on whether mutex is of ERRORCHECK type or not.
	 */
	if (upimutex == NULL || upimutex->upi_owner != curthread) {
		mutex_exit(&upibp->upib_lock);
		error = EPERM;
		goto out;
	}
	mutex_exit(&upibp->upib_lock); /* release for user memory access */
	upilocked = 1;
	fuword16_noerr(&lp->mutex_flag, &flag);
	if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
		/*
		 * transition mutex to the LOCK_NOTRECOVERABLE state.
		 */
		flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
		flag |= LOCK_NOTRECOVERABLE;
		suword16_noerr(&lp->mutex_flag, flag);
	}
	if (type & USYNC_PROCESS)
		suword32_noerr(&lp->mutex_ownerpid, 0);
	upimutex_unlock((upimutex_t *)upimutex, flag);
	upilocked = 0;
out:
	no_fault();
	return (error);
}

/*
 * Clear the contents of a user-level mutex; return the flags.
 * Used only by upi_dead() and lwp_mutex_cleanup(), below.
 */
static uint16_t
lwp_clear_mutex(lwp_mutex_t *lp, uint16_t lockflg)
{
	uint16_t flag;

	fuword16_noerr(&lp->mutex_flag, &flag);
	if ((flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) == 0) {
		flag |= lockflg;
		suword16_noerr(&lp->mutex_flag, flag);
	}
	suword32_noerr((uint32_t *)&lp->mutex_owner, 0);
	suword32_noerr((uint32_t *)&lp->mutex_owner + 1, 0);
	suword32_noerr(&lp->mutex_ownerpid, 0);
	suword8_noerr(&lp->mutex_rcount, 0);

	return (flag);
}

/*
 * Mark user mutex state, corresponding to kernel upimutex,
 * as LOCK_UNMAPPED or LOCK_OWNERDEAD, as appropriate
 */
static int
upi_dead(upimutex_t *upip, uint16_t lockflg)
{
	label_t ljb;
	int error = 0;
	lwp_mutex_t *lp;

	if (