xref: /netvirt/usr/src/uts/common/syscall/poll.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved  	*/

/*
 * Portions of this source code were derived from Berkeley 4.3 BSD
 * under license from the Regents of the University of California.
 */

#pragma ident	"@(#)poll.c	1.133	07/10/25 SMI"

#include <sys/param.h>
#include <sys/isa_defs.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/user.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/time.h>
#include <sys/vnode.h>
#include <sys/file.h>
#include <sys/mode.h>
#include <sys/proc.h>
#include <sys/uio.h>
#include <sys/poll_impl.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/bitmap.h>
#include <sys/kstat.h>
#include <sys/rctl.h>
#include <sys/port_impl.h>
#include <sys/schedctl.h>

#define	NPHLOCKS	64	/* Number of locks; must be power of 2 */
#define	PHLOCKADDR(php)	&plocks[(((uintptr_t)(php)) >> 8) & (NPHLOCKS - 1)]
#define	PHLOCK(php)	PHLOCKADDR(php).pp_lock
#define	PH_ENTER(php)	mutex_enter(PHLOCK(php))
#define	PH_EXIT(php)	mutex_exit(PHLOCK(php))
#define	VALID_POLL_EVENTS	(POLLIN | POLLPRI | POLLOUT | POLLRDNORM \
	| POLLRDBAND | POLLWRBAND | POLLHUP | POLLERR | POLLNVAL)

/*
 * global counters to collect some stats
 */
static struct {
	kstat_named_t	polllistmiss;	/* failed to find a cached poll list */
	kstat_named_t	pollcachehit;	/* list matched 100% w/ cached one */
	kstat_named_t	pollcachephit;	/* list matched < 100% w/ cached one */
	kstat_named_t	pollcachemiss;	/* every list entry is dif from cache */
} pollstats = {
	{ "polllistmiss",	KSTAT_DATA_UINT64 },
	{ "pollcachehit",	KSTAT_DATA_UINT64 },
	{ "pollcachephit",	KSTAT_DATA_UINT64 },
	{ "pollcachemiss",	KSTAT_DATA_UINT64 }
};

kstat_named_t *pollstats_ptr = (kstat_named_t *)&pollstats;
uint_t pollstats_ndata = sizeof (pollstats) / sizeof (kstat_named_t);

struct pplock	{
	kmutex_t	pp_lock;
	short		pp_flag;
	kcondvar_t	pp_wait_cv;
	int32_t		pp_pad;		/* to a nice round 16 bytes */
};

static struct pplock plocks[NPHLOCKS];	/* Hash array of pollhead locks */

#ifdef DEBUG
static int pollchecksanity(pollstate_t *, nfds_t);
static int pollcheckxref(pollstate_t *, int);
static void pollcheckphlist(void);
static int pollcheckrevents(pollstate_t *, int, int, int);
static void checkpolldat(pollstate_t *);
#endif	/* DEBUG */
static int plist_chkdupfd(file_t *, polldat_t *, pollstate_t *, pollfd_t *, int,
    int *);

/*
 * Data structure overview:
 * The per-thread poll state consists of
 *	one pollstate_t
 *	one pollcache_t
 *	one bitmap with one event bit per fd
 *	a (two-dimensional) hashed array of polldat_t structures - one entry
 *	per fd
 *
 * This conglomerate of data structures interact with
 *	the pollhead which is used by VOP_POLL and pollwakeup
 *	(protected by the PHLOCK, cached array of plocks), and
 *	the fpollinfo list hanging off the fi_list which is used to notify
 *	poll when a cached fd is closed. This is protected by uf_lock.
 *
 * Invariants:
 *	pd_php (pollhead pointer) is set iff (if and only if) the polldat
 *	is on that pollhead. This is modified atomically under pc_lock.
 *
 *	pd_fp (file_t pointer) is set iff the thread is on the fpollinfo
 *	list for that open file.
 *	This is modified atomically under pc_lock.
 *
 *	pd_count is the sum (over all values of i) of pd_ref[i].xf_refcnt.
 *	Iff pd_ref[i].xf_refcnt >= 1 then
 *		ps_pcacheset[i].pcs_pollfd[pd_ref[i].xf_position].fd == pd_fd
 *	Iff pd_ref[i].xf_refcnt > 1 then
 *		In ps_pcacheset[i].pcs_pollfd between index
 *		pd_ref[i].xf_position] and the end of the list
 *		there are xf_refcnt entries with .fd == pd_fd
 *
 * Locking design:
 * Whenever possible the design relies on the fact that the poll cache state
 * is per thread thus for both poll and exit it is self-synchronizing.
 * Thus the key interactions where other threads access the state are:
 *	pollwakeup (and polltime), and
 *	close cleaning up the cached references to an open file
 *
 * The two key locks in poll proper is ps_lock and pc_lock.
 *
 * The ps_lock is used for synchronization between poll, (lwp_)exit and close
 * to ensure that modifications to pollcacheset structure are serialized.
 * This lock is held through most of poll() except where poll sleeps
 * since there is little need to handle closes concurrently with the execution
 * of poll.
 * The pc_lock protects most of the fields in pollcache structure and polldat
 * structures (which are accessed by poll, pollwakeup, and polltime)
 * with the exception of fields that are only modified when only one thread
 * can access this per-thread state.
 * Those exceptions occur in poll when first allocating the per-thread state,
 * when poll grows the number of polldat (never shrinks), and when
 * exit/pollcleanup has ensured that there are no references from either
 * pollheads or fpollinfo to the threads poll state.
 *
 * Poll(2) system call is the only path which ps_lock and pc_lock are both
 * held, in that order. It needs ps_lock to synchronize with close and
 * lwp_exit; and pc_lock with pollwakeup.
 *
 * The locking interaction between pc_lock and PHLOCK take into account
 * that poll acquires these locks in the order of pc_lock and then PHLOCK
 * while pollwakeup does it in the reverse order. Thus pollwakeup implements
 * deadlock avoidance by dropping the locks and reacquiring them in the
 * reverse order. For this to work pollwakeup needs to prevent the thread
 * from exiting and freeing all of the poll related state. Thus is done
 * using
 *	the pc_no_exit lock
 *	the pc_busy counter
 *	the pc_busy_cv condition variable
 *
 * The locking interaction between pc_lock and uf_lock has similar
 * issues. Poll holds ps_lock and/or pc_lock across calls to getf/releasef
 * which acquire uf_lock. The poll cleanup in close needs to hold uf_lock
 * to prevent poll or exit from doing a delfpollinfo after which the thread
 * might exit. But the cleanup needs to acquire pc_lock when modifying
 * the poll cache state. The solution is to use pc_busy and do the close
 * cleanup in two phases:
 *	First close calls pollblockexit which increments pc_busy.
 *	This prevents the per-thread poll related state from being freed.
 *	Then close drops uf_lock and calls pollcacheclean.
 *	This routine can then acquire pc_lock and remove any references
 *	to the closing fd (as well as recording that it has been closed
 *	so that a POLLNVAL can be generated even if the fd is reused before
 *	poll has been woken up and checked getf() again).
 *
 * When removing a polled fd from poll cache, the fd is always removed
 * from pollhead list first and then from fpollinfo list, i.e.,
 * pollhead_delete() is called before delfpollinfo().
 *
 *
 * Locking hierarchy:
 *	pc_no_exit is a leaf level lock.
 *	ps_lock is held when acquiring pc_lock (except when pollwakeup
 *	acquires pc_lock).
 *	pc_lock might be held when acquiring PHLOCK (pollhead_insert/
 *	pollhead_delete)
 *	pc_lock is always held (but this is not required)
 *	when acquiring PHLOCK (in polladd/pollhead_delete and pollwakeup called
 *	from pcache_clean_entry).
 *	pc_lock is held across addfpollinfo/delfpollinfo which acquire
 *	uf_lock.
 *	pc_lock is held across getf/releasef which acquire uf_lock.
 *	ps_lock might be held across getf/releasef which acquire uf_lock.
 *	pollwakeup tries to acquire pc_lock while holding PHLOCK
 *	but drops the locks and reacquire them in reverse order to avoid
 *	deadlock.
 *
 * Note also that there is deadlock avoidance support for VOP_POLL routines
 * and pollwakeup involving a file system or driver lock.
 * See below.
 */

/*
 * Deadlock avoidance support for VOP_POLL() routines.  This is
 * sometimes necessary to prevent deadlock between polling threads
 * (which hold poll locks on entry to xx_poll(), then acquire foo)
 * and pollwakeup() threads (which hold foo, then acquire poll locks).
 *
 * pollunlock(void) releases whatever poll locks the current thread holds,
 *	returning a cookie for use by pollrelock();
 *
 * pollrelock(cookie) reacquires previously dropped poll locks;
 *
 * polllock(php, mutex) does the common case: pollunlock(),
 *	acquire the problematic mutex, pollrelock().
 */
int
pollunlock(void)
{
	pollcache_t *pcp;
	int lockstate = 0;

	/*
	 * t_pollcache is set by /dev/poll and event ports (port_fd.c).
	 * If the pollrelock/pollunlock is called as a result of poll(2),
	 * the t_pollcache should be NULL.
	 */
	if (curthread->t_pollcache == NULL)
		pcp = curthread->t_pollstate->ps_pcache;
	else
		pcp = curthread->t_pollcache;

	if (mutex_owned(&pcp->pc_lock)) {
		lockstate = 1;
		mutex_exit(&pcp->pc_lock);
	}
	return (lockstate);
}

void
pollrelock(int lockstate)
{
	pollcache_t *pcp;

	/*
	 * t_pollcache is set by /dev/poll and event ports (port_fd.c).
	 * If the pollrelock/pollunlock is called as a result of poll(2),
	 * the t_pollcache should be NULL.
	 */
	if (curthread->t_pollcache == NULL)
		pcp = curthread->t_pollstate->ps_pcache;
	else
		pcp = curthread->t_pollcache;

	if (lockstate > 0)
		mutex_enter(&pcp->pc_lock);
}

/* ARGSUSED */
void
polllock(pollhead_t *php, kmutex_t *lp)
{
	if (!mutex_tryenter(lp)) {
		int lockstate = pollunlock();
		mutex_enter(lp);
		pollrelock(lockstate);
	}
}

static int
poll_common(pollfd_t *fds, nfds_t nfds, timespec_t *tsp, k_sigset_t *ksetp)
{
	kthread_t *t = curthread;
	klwp_t *lwp = ttolwp(t);
	proc_t *p = ttoproc(t);
	int fdcnt = 0;
	int rval;
	int i;
	timespec_t *rqtp = NULL;
	int timecheck = 0;
	int imm_timeout = 0;
	pollfd_t *pollfdp;
	pollstate_t *ps;
	pollcache_t *pcp;
	int error = 0;
	nfds_t old_nfds;
	int cacheindex = 0;	/* which cache set is used */

	/*
	 * Determine the precise future time of the requested timeout, if any.
	 */
	if (tsp != NULL) {
		if (tsp->tv_sec == 0 && tsp->tv_nsec == 0)
			imm_timeout = 1;
		else {
			timespec_t now;
			timecheck = timechanged;
			gethrestime(&now);
			rqtp = tsp;
			timespecadd(rqtp, &now);
		}
	}

	/*
	 * Reset our signal mask, if requested.
	 */
	if (ksetp != NULL) {
		mutex_enter(&p->p_lock);
		schedctl_finish_sigblock(t);
		lwp->lwp_sigoldmask = t->t_hold;
		t->t_hold = *ksetp;
		t->t_flag |= T_TOMASK;
		/*
		 * Call cv_timedwait_sig() just to check for signals.
		 * We will return immediately with either 0 or -1.
		 */
		if (!cv_timedwait_sig(&t->t_delay_cv, &p->p_lock, lbolt)) {
			mutex_exit(&p->p_lock);
			error = EINTR;
			goto pollout;
		}
		mutex_exit(&p->p_lock);
	}

	/*
	 * Check to see if this guy just wants to use poll() as a timeout.
	 * If yes then bypass all the other stuff and make him sleep.
	 */
	if (nfds == 0) {
		/*
		 * Sleep until we have passed the requested future
		 * time or until interrupted by a signal.
		 * Do not check for signals if we have a zero timeout.
		 */
		if (!imm_timeout) {
			mutex_enter(&t->t_delay_lock);
			while ((rval = cv_waituntil_sig(&t->t_delay_cv,
			    &t->t_delay_lock, rqtp, timecheck)) > 0)
				continue;
			mutex_exit(&t->t_delay_lock);
			if (rval == 0)
				error = EINTR;
		}
		goto pollout;
	}

	if (nfds > p->p_fno_ctl) {
		mutex_enter(&p->p_lock);
		(void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
		    p->p_rctls, p, RCA_SAFE);
		mutex_exit(&p->p_lock);
		error = EINVAL;
		goto pollout;
	}

	/*
	 * Need to allocate memory for pollstate before anything because
	 * the mutex and cv are created in this space
	 */
	if ((ps = t->t_pollstate) == NULL) {
		t->t_pollstate = pollstate_create();
		ps = t->t_pollstate;
	}

	if (ps->ps_pcache == NULL)
		ps->ps_pcache = pcache_alloc();
	pcp = ps->ps_pcache;

	/*
	 * NOTE: for performance, buffers are saved across poll() calls.
	 * The theory is that if a process polls heavily, it tends to poll
	 * on the same set of descriptors.  Therefore, we only reallocate
	 * buffers when nfds changes.  There is no hysteresis control,
	 * because there is no data to suggest that this is necessary;
	 * the penalty of reallocating is not *that* great in any event.
	 */
	old_nfds = ps->ps_nfds;
	if (nfds != old_nfds) {

		kmem_free(ps->ps_pollfd, old_nfds * sizeof (pollfd_t));
		pollfdp = kmem_alloc(nfds * sizeof (pollfd_t), KM_SLEEP);
		ps->ps_pollfd = pollfdp;
		ps->ps_nfds = nfds;
	}

	pollfdp = ps->ps_pollfd;
	if (copyin(fds, pollfdp, nfds * sizeof (pollfd_t))) {
		error = EFAULT;
		goto pollout;
	}

	if (fds == NULL) {
		/*
		 * If the process has page 0 mapped, then the copyin() above
		 * will succeed even if fds is NULL.  However, our cached
		 * poll lists are keyed by the address of the passed-in fds
		 * structure, and we use the value NULL to indicate an unused
		 * poll cache list entry.  As such, we elect not to support
		 * NULL as a valid (user) memory address and fail the poll()
		 * call.
		 */
		error = EINVAL;
		goto pollout;
	}

	/*
	 * If this thread polls for the first time, allocate ALL poll
	 * cache data structures and cache the poll fd list. This
	 * allocation is delayed till now because lwp's polling 0 fd
	 * (i.e. using poll as timeout()) don't need this memory.
	 */
	mutex_enter(&ps->ps_lock);
	pcp = ps->ps_pcache;
	ASSERT(pcp != NULL);
	if (pcp->pc_bitmap == NULL) {
		pcache_create(pcp, nfds);
		/*
		 * poll and cache this poll fd list in ps_pcacheset[0].
		 */
		error = pcacheset_cache_list(ps, fds, &fdcnt, cacheindex);
		if (fdcnt || error) {
			mutex_exit(&ps->ps_lock);
			goto pollout;
		}
	} else {
		pollcacheset_t	*pcset = ps->ps_pcacheset;

		/*
		 * Not first time polling. Select a cached poll list by
		 * matching user pollfd list buffer address.
		 */
		for (cacheindex = 0; cacheindex < ps->ps_nsets; cacheindex++) {
			if (pcset[cacheindex].pcs_usradr == (uintptr_t)fds) {
				if ((++pcset[cacheindex].pcs_count) == 0) {
					/*
					 * counter is wrapping around.
					 */
					pcacheset_reset_count(ps, cacheindex);
				}
				/*
				 * examine and resolve possible
				 * difference of the current poll
				 * list and previously cached one.
				 * If there is an error during resolve(),
				 * the callee will guarantee the consistency
				 * of cached poll list and cache content.
				 */
				error = pcacheset_resolve(ps, nfds, &fdcnt,
				    cacheindex);
				if (error) {
					mutex_exit(&ps->ps_lock);
					goto pollout;
				}
				break;
			}

			/*
			 * Note that pcs_usradr field of an used entry won't be
			 * NULL because it stores the address of passed-in fds,
			 * and NULL fds will not be cached (Then it is either
			 * the special timeout case when nfds is 0 or it returns
			 * failure directly).
			 */
			if (pcset[cacheindex].pcs_usradr == NULL) {
				/*
				 * found an unused entry. Use it to cache
				 * this poll list.
				 */
				error = pcacheset_cache_list(ps, fds, &fdcnt,
				    cacheindex);
				if (fdcnt || error) {
					mutex_exit(&ps->ps_lock);
					goto pollout;
				}
				break;
			}
		}
		if (cacheindex == ps->ps_nsets) {
			/*
			 * We failed to find a matching cached poll fd list.
			 * replace an old list.
			 */
			pollstats.polllistmiss.value.ui64++;
			cacheindex = pcacheset_replace(ps);
			ASSERT(cacheindex < ps->ps_nsets);
			pcset[cacheindex].pcs_usradr = (uintptr_t)fds;
			error = pcacheset_resolve(ps, nfds, &fdcnt, cacheindex);
			if (error) {
				mutex_exit(&ps->ps_lock);
				goto pollout;
			}
		}
	}

	/*
	 * Always scan the bitmap with the lock on the pollcache held.
	 * This is to make sure that a wakeup does not come undetected.
	 * If the lock is not held, a pollwakeup could have come for an
	 * fd we already checked but before this thread sleeps, in which
	 * case the wakeup is missed. Now we hold the pcache lock and
	 * check the bitmap again. This will prevent wakeup from happening
	 * while we hold pcache lock since pollwakeup() will also lock
	 * the pcache before updating poll bitmap.
	 */
	mutex_enter(&pcp->pc_lock);
	for (;;) {
		pcp->pc_flag = 0;
		error = pcache_poll(pollfdp, ps, nfds, &fdcnt, cacheindex);
		if (fdcnt || error) {
			mutex_exit(&pcp->pc_lock);
			mutex_exit(&ps->ps_lock);
			break;
		}

		/*
		 * If T_POLLWAKE is set, a pollwakeup() was performed on
		 * one of the file descriptors.  This can happen only if
		 * one of the VOP_POLL() functions dropped pcp->pc_lock.
		 * The only current cases of this is in procfs (prpoll())
		 * and STREAMS (strpoll()).
		 */
		if (pcp->pc_flag & T_POLLWAKE)
			continue;

		/*
		 * If you get here, the poll of fds was unsuccessful.
		 * Wait until some fd becomes readable, writable, or gets
		 * an exception, or until a signal or a timeout occurs.
		 * Do not check for signals if we have a zero timeout.
		 */
		mutex_exit(&ps->ps_lock);
		if (imm_timeout)
			rval = -1;
		else
			rval = cv_waituntil_sig(&pcp->pc_cv, &pcp->pc_lock,
				rqtp, timecheck);
		mutex_exit(&pcp->pc_lock);
		/*
		 * If we have received a signal or timed out
		 * then break out and return.
		 */
		if (rval <= 0) {
			if (rval == 0)
				error = EINTR;
			break;
		}
		/*
		 * We have not received a signal or timed out.
		 * Continue around and poll fds again.
		 */
		mutex_enter(&ps->ps_lock);
		mutex_enter(&pcp->pc_lock);
	}

pollout:
	/*
	 * If we changed the signal mask but we received
	 * no signal then restore the signal mask.
	 * Otherwise psig() will deal with the signal mask.
	 */
	if (ksetp != NULL) {
		mutex_enter(&p->p_lock);
		if (lwp->lwp_cursig == 0) {
			t->t_hold = lwp->lwp_sigoldmask;
			t->t_flag &= ~T_TOMASK;
		}
		mutex_exit(&p->p_lock);
	}

	if (error)
		return (set_errno(error));

	/*
	 * Copy out the events and return the fdcnt to the user.
	 */
	if (nfds != 0 &&
	    copyout(pollfdp, fds, nfds * sizeof (pollfd_t)))
		return (set_errno(EFAULT));

#ifdef DEBUG
	/*
	 * Another sanity check:
	 */
	if (fdcnt) {
		int	reventcnt = 0;

		for (i = 0; i < nfds; i++) {
			if (pollfdp[i].fd < 0) {
				ASSERT(pollfdp[i].revents == 0);
				continue;
			}
			if (pollfdp[i].revents) {
				reventcnt++;
			}
		}
		ASSERT(fdcnt == reventcnt);
	} else {
		for (i = 0; i < nfds; i++) {
			ASSERT(pollfdp[i].revents == 0);
		}
	}
#endif	/* DEBUG */

	return (fdcnt);
}

/*
 * This system call trap exists solely for binary compatibility with
 * old statically-linked applications.  It is not called from libc.
 * It should be removed in the next release.
 */
int
poll(pollfd_t *fds, nfds_t nfds, int time_out)
{
	timespec_t ts;
	timespec_t *tsp;

	if (time_out < 0)
		tsp = NULL;
	else {
		ts.tv_sec = time_out / MILLISEC;
		ts.tv_nsec = (time_out % MILLISEC) * MICROSEC;
		tsp = &ts;
	}

	return (poll_common(fds, nfds, tsp, NULL));
}

/*
 * This is the system call trap that poll(),
 * select() and pselect() are built upon.
 * It is a private interface between libc and the kernel.
 */
int
pollsys(pollfd_t *fds, nfds_t nfds, timespec_t *timeoutp, sigset_t *setp)
{
	timespec_t ts;
	timespec_t *tsp;
	sigset_t set;
	k_sigset_t kset;
	k_sigset_t *ksetp;
	model_t datamodel = get_udatamodel();

	if (timeoutp == NULL)
		tsp = NULL;
	else {
		if (datamodel == DATAMODEL_NATIVE) {
			if (copyin(timeoutp, &ts, sizeof (ts)))
				return (set_errno(EFAULT));
		} else {
			timespec32_t ts32;

			if (copyin(timeoutp, &ts32, sizeof (ts32)))
				return (set_errno(EFAULT));
			TIMESPEC32_TO_TIMESPEC(&ts, &ts32)
		}

		if (itimerspecfix(&ts))
			return (set_errno(EINVAL));
		tsp = &ts;
	}

	if (setp == NULL)
		ksetp = NULL;
	else {
		if (copyin(setp, &set, sizeof (set)))
			return (set_errno(EFAULT));
		sigutok(&set, &kset);
		ksetp = &kset;
	}

	return (poll_common(fds, nfds, tsp, ksetp));
}

/*
 * Clean up any state left around by poll(2). Called when a thread exits.
 */
void
pollcleanup()
{
	pollstate_t *ps = curthread->t_pollstate;
	pollcache_t *pcp;

	if (ps == NULL)
		return;
	pcp = ps->ps_pcache;
	/*
	 * free up all cached poll fds
	 */
	if (pcp == NULL) {
		/* this pollstate is used by /dev/poll */
		goto pollcleanout;
	}

	if (pcp->pc_bitmap != NULL) {
		ASSERT(MUTEX_NOT_HELD(&ps->ps_lock));
		/*
		 * a close lwp can race with us when cleaning up a polldat
		 * entry. We hold the ps_lock when cleaning hash table.
		 * Since this pollcache is going away anyway, there is no
		 * need to hold the pc_lock.
		 */
		mutex_enter(&ps->ps_lock);
		pcache_clean(pcp);
		mutex_exit(&ps->ps_lock);
#ifdef DEBUG
		/*
		 * At this point, all fds cached by this lwp should be
		 * cleaned up. There should be no fd in fi_list still
		 * reference this thread.
		 */
		checkfpollinfo();	/* sanity check */
		pollcheckphlist();	/* sanity check */
#endif	/* DEBUG */
	}
	/*
	 * Be sure no one is referencing thread before exiting
	 */
	mutex_enter(&pcp->pc_no_exit);
	ASSERT(pcp->pc_busy >= 0);
	while (pcp->pc_busy > 0)
		cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
	mutex_exit(&pcp->pc_no_exit);
pollcleanout:
	pollstate_destroy(ps);
	curthread->t_pollstate = NULL;
}

/*
 * pollwakeup() - poke threads waiting in poll() for some event
 * on a particular object.
 *
 * The threads hanging off of the specified pollhead structure are scanned.
 * If their event mask matches the specified event(s), then pollnotify() is
 * called to poke the thread.
 *
 * Multiple events may be specified.  When POLLHUP or POLLERR are specified,
 * all waiting threads are poked.
 *
 * It is important that pollnotify() not drop the lock protecting the list
 * of threads.
 */
void
pollwakeup(pollhead_t *php, short events_arg)
{
	polldat_t	*pdp;
	int		events = (ushort_t)events_arg;
	struct plist {
		port_t *pp;
		int	pevents;
		struct plist *next;
		};
	struct plist *plhead = NULL, *pltail = NULL;

retry:
	PH_ENTER(php);

	for (pdp = php->ph_list; pdp; pdp = pdp->pd_next) {
		if ((pdp->pd_events & events) ||
		    (events & (POLLHUP | POLLERR))) {

			pollcache_t 	*pcp;

			if (pdp->pd_portev != NULL) {
				port_kevent_t	*pkevp = pdp->pd_portev;
				/*
				 * Object (fd) is associated with an event port,
				 * => send event notification to the port.
				 */
				ASSERT(pkevp->portkev_source == PORT_SOURCE_FD);
				mutex_enter(&pkevp->portkev_lock);
				if (pkevp->portkev_flags & PORT_KEV_VALID) {
					int pevents;

					pkevp->portkev_flags &= ~PORT_KEV_VALID;
					pkevp->portkev_events |= events &
					    (pdp->pd_events | POLLHUP |
					    POLLERR);
					/*
					 * portkev_lock mutex will be released
					 * by port_send_event().
					 */
					port_send_event(pkevp);

					/*
					 * If we have some thread polling the
					 * port's fd, add it to the list. They
					 * will be notified later.
					 * The port_pollwkup() will flag the
					 * port_t so that it will not disappear
					 * till port_pollwkdone() is called.
					 */
					pevents =
					    port_pollwkup(pkevp->portkev_port);
					if (pevents) {
						struct plist *t;
						t = kmem_zalloc(
							sizeof (struct plist),
							    KM_SLEEP);
						t->pp = pkevp->portkev_port;
						t->pevents = pevents;
						if (plhead == NULL) {
							plhead = t;
						} else {
							pltail->next = t;
						}
						pltail = t;
					}
				} else {
					mutex_exit(&pkevp->portkev_lock);
				}
				continue;
			}

			pcp = pdp->pd_pcache;

			/*
			 * Try to grab the lock for this thread. If
			 * we don't get it then we may deadlock so
			 * back out and restart all over again. Note
			 * that the failure rate is very very low.
			 */
			if (mutex_tryenter(&pcp->pc_lock)) {
				pollnotify(pcp, pdp->pd_fd);
				mutex_exit(&pcp->pc_lock);
			} else {
				/*
				 * We are here because:
				 *	1) This thread has been woke up
				 *	   and is trying to get out of poll().
				 *	2) Some other thread is also here
				 *	   but with a different pollhead lock.
				 *
				 * So, we need to drop the lock on pollhead
				 * because of (1) but we want to prevent
				 * that thread from doing lwp_exit() or
				 * devpoll close. We want to ensure that
				 * the pollcache pointer is still invalid.
				 *
				 * Solution: Grab the pcp->pc_no_exit lock,
				 * increment the pc_busy counter, drop every
				 * lock in sight. Get out of the way and wait
				 * for type (2) threads to finish.
				 */

				mutex_enter(&pcp->pc_no_exit);
				pcp->pc_busy++;	/* prevents exit()'s */
				mutex_exit(&pcp->pc_no_exit);

				PH_EXIT(php);
				mutex_enter(&pcp->pc_lock);
				mutex_exit(&pcp->pc_lock);
				mutex_enter(&pcp->pc_no_exit);
				pcp->pc_busy--;
				if (pcp->pc_busy == 0) {
					/*
					 * Wakeup the thread waiting in
					 * thread_exit().
					 */
					cv_signal(&pcp->pc_busy_cv);
				}
				mutex_exit(&pcp->pc_no_exit);
				goto retry;
			}
		}
	}


	/*
	 * Event ports - If this php is of the port on the list,
	 * call port_pollwkdone() to release it. The port_pollwkdone()
	 * needs to be called before dropping the PH lock so that any new
	 * thread attempting to poll this port are blocked. There can be
	 * only one thread here in pollwakeup notifying this port's fd.
	 */
	if (plhead != NULL && &plhead->pp->port_pollhd == php) {
		struct plist *t;
		port_pollwkdone(plhead->pp);
		t = plhead;
		plhead = plhead->next;
		kmem_free(t, sizeof (struct plist));
	}
	PH_EXIT(php);

	/*
	 * Event ports - Notify threads polling the event port's fd.
	 * This is normally done in port_send_event() where it calls
	 * pollwakeup() on the port. But, for PORT_SOURCE_FD source alone,
	 * we do it here in pollwakeup() to avoid a recursive call.
	 */
	if (plhead != NULL) {
		php = &plhead->pp->port_pollhd;
		events = plhead->pevents;
		goto retry;
	}
}

/*
 * This function is called to inform a thread that
 * an event being polled for has occurred.
 * The pollstate lock on the thread should be held on entry.
 */
void
pollnotify(pollcache_t *pcp, int fd)
{
	ASSERT(fd < pcp->pc_mapsize);
	ASSERT(MUTEX_HELD(&pcp->pc_lock));
	BT_SET(pcp->pc_bitmap, fd);
	pcp->pc_flag |= T_POLLWAKE;
	cv_signal(&pcp->pc_cv);
}

/*
 * add a polldat entry to pollhead ph_list. The polldat struct is used
 * by pollwakeup to wake sleeping pollers when polled events has happened.
 */
void
pollhead_insert(pollhead_t *php, polldat_t *pdp)
{
	PH_ENTER(php);
	ASSERT(pdp->pd_next == NULL);
#ifdef DEBUG
	{
		/*
		 * the polldat should not be already on the list
		 */
		polldat_t *wp;
		for (wp = php->ph_list; wp; wp = wp->pd_next) {
			ASSERT(wp != pdp);
		}
	}
#endif	/* DEBUG */
	pdp->pd_next = php->ph_list;
	php->ph_list = pdp;
	PH_EXIT(php);
}

/*
 * Delete the polldat entry from ph_list.
 */
void
pollhead_delete(pollhead_t *php, polldat_t *pdp)
{
	polldat_t *wp;
	polldat_t **wpp;

	PH_ENTER(php);
	for (wpp = &php->ph_list; (wp = *wpp) != NULL; wpp = &wp->pd_next) {
		if (wp == pdp) {
			*wpp = pdp->pd_next;
			pdp->pd_next = NULL;
			break;
		}
	}
#ifdef DEBUG
	/* assert that pdp is no longer in the list */
	for (wp = *wpp; wp; wp = wp->pd_next) {
		ASSERT(wp != pdp);
	}
#endif	/* DEBUG */
	PH_EXIT(php);
}

/*
 * walk through the poll fd lists to see if they are identical. This is an
 * expensive operation and should not be done more than once for each poll()
 * call.
 *
 * As an optimization (i.e., not having to go through the lists more than
 * once), this routine also clear the revents field of pollfd in 'current'.
 * Zeroing out the revents field of each entry in current poll list is
 * required by poll man page.
 *
 * Since the events field of cached list has illegal poll events filtered
 * out, the current list applies the same filtering before comparison.
 *
 * The routine stops when it detects a meaningful difference, or when it
 * exhausts the lists.
 */
int
pcacheset_cmp(pollfd_t *current, pollfd_t *cached, pollfd_t *newlist, int n)
{
	int    ix;

	for (ix = 0; ix < n; ix++) {
		if (current[ix].fd == cached[ix].fd) {
			/*
			 * Filter out invalid poll events while we are in
			 * inside the loop.
			 */
			if (current[ix].events & ~VALID_POLL_EVENTS) {
				current[ix].events &= VALID_POLL_EVENTS;
				if (newlist != NULL)
					newlist[ix].events = current[ix].events;
			}
			if (current[ix].events == cached[ix].events) {
				current[ix].revents = 0;
				continue;
			}
		}
		if ((current[ix].fd < 0) && (cached[ix].fd < 0)) {
			current[ix].revents = 0;
			continue;
		}
		return (ix);
	}
	return (ix);
}

/*
 * This routine returns a pointer to a cached poll fd entry, or NULL if it
 * does not find it in the hash table.
 */
polldat_t *
pcache_lookup_fd(pollcache_t *pcp, int fd)
{
	int hashindex;
	polldat_t *pdp;

	hashindex = POLLHASH(pcp->pc_hashsize, fd);
	pdp = pcp->pc_hash[hashindex];
	while (pdp != NULL) {
		if (pdp->pd_fd == fd)
			break;
		pdp = pdp->pd_hashnext;
	}
	return (pdp);
}

polldat_t *
pcache_alloc_fd(int nsets)
{
	polldat_t *pdp;

	pdp = kmem_zalloc(sizeof (polldat_t), KM_SLEEP);
	if (nsets > 0) {
		pdp->pd_ref = kmem_zalloc(sizeof (xref_t) * nsets, KM_SLEEP);
		pdp->pd_nsets = nsets;
	}
	return (pdp);
}

/*
 * This routine  inserts a polldat into the pollcache's hash table. It
 * may be necessary to grow the size of the hash table.
 */
void
pcache_insert_fd(pollcache_t *pcp, polldat_t *pdp, nfds_t nfds)
{
	int hashindex;
	int fd;

	if ((pcp->pc_fdcount > pcp->pc_hashsize * POLLHASHTHRESHOLD) ||
	    (nfds > pcp->pc_hashsize * POLLHASHTHRESHOLD)) {
		pcache_grow_hashtbl(pcp, nfds);
	}
	fd = pdp->pd_fd;
	hashindex = POLLHASH(pcp->pc_hashsize, fd);
	pdp->pd_hashnext = pcp->pc_hash[hashindex];
	pcp->pc_hash[hashindex] = pdp;
	pcp->pc_fdcount++;

#ifdef DEBUG
	{
		/*
		 * same fd should not appear on a hash list twice
		 */
		polldat_t *pdp1;
		for (pdp1 = pdp->pd_hashnext; pdp1; pdp1 = pdp1->pd_hashnext) {
			ASSERT(pdp->pd_fd != pdp1->pd_fd);
		}
	}
#endif	/* DEBUG */
}

/*
 * Grow the hash table -- either double the table size or round it to the
 * nearest multiples of POLLHASHCHUNKSZ, whichever is bigger. Rehash all the
 * elements on the hash table.
 */
void
pcache_grow_hashtbl(pollcache_t *pcp, nfds_t nfds)
{
	int	oldsize;
	polldat_t **oldtbl;
	polldat_t *pdp, *pdp1;
	int	i;
#ifdef DEBUG
	int	count = 0;
#endif

	ASSERT(pcp->pc_hashsize % POLLHASHCHUNKSZ == 0);
	oldsize = pcp->pc_hashsize;
	oldtbl = pcp->pc_hash;
	if (nfds > pcp->pc_hashsize * POLLHASHINC) {
		pcp->pc_hashsize = (nfds + POLLHASHCHUNKSZ - 1) &
		    ~(POLLHASHCHUNKSZ - 1);
	} else {
		pcp->pc_hashsize = pcp->pc_hashsize * POLLHASHINC;
	}
	pcp->pc_hash = kmem_zalloc(pcp->pc_hashsize * sizeof (polldat_t *),
	    KM_SLEEP);
	/*
	 * rehash existing elements
	 */
	pcp->pc_fdcount = 0;
	for (i = 0; i < oldsize; i++) {
		pdp = oldtbl[i];
		while (pdp != NULL) {
			pdp1 = pdp->pd_hashnext;
			pcache_insert_fd(pcp, pdp, nfds);
			pdp = pdp1;
#ifdef DEBUG
			count++;
#endif
		}
	}
	kmem_free(oldtbl, oldsize * sizeof (polldat_t *));
	ASSERT(pcp->pc_fdcount == count);
}

void
pcache_grow_map(pollcache_t *pcp, int fd)
{
	int  	newsize;
	ulong_t	*newmap;

	/*
	 * grow to nearest multiple of POLLMAPCHUNK, assuming POLLMAPCHUNK is
	 * power of 2.
	 */
	newsize = (fd + POLLMAPCHUNK) & ~(POLLMAPCHUNK - 1);
	newmap = kmem_zalloc((newsize / BT_NBIPUL) * sizeof (ulong_t),
	    KM_SLEEP);
	/*
	 * don't want pollwakeup to set a bit while growing the bitmap.
	 */
	ASSERT(mutex_owned(&pcp->pc_lock) == 0);
	mutex_enter(&pcp->pc_lock);
	bcopy(pcp->pc_bitmap, newmap,
	    (pcp->pc_mapsize / BT_NBIPUL) * sizeof (ulong_t));
	kmem_free(pcp->pc_bitmap,
	    (pcp->pc_mapsize /BT_NBIPUL) * sizeof (ulong_t));
	pcp->pc_bitmap = newmap;
	pcp->pc_mapsize = newsize;
	mutex_exit(&pcp->pc_lock);
}

/*
 * remove all the reference from pollhead list and fpollinfo lists.
 */
void
pcache_clean(pollcache_t *pcp)
{
	int i;
	polldat_t **hashtbl;
	polldat_t *pdp;

	ASSERT(MUTEX_HELD(&curthread->t_pollstate->ps_lock));
	hashtbl = pcp->pc_hash;
	for (i = 0; i < pcp->pc_hashsize; i++) {
		for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
			if (pdp->pd_php != NULL) {