xref: /netvirt/usr/src/uts/common/vm/vm_page.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989  AT&T	*/
/*	  All Rights Reserved  	*/

/*
 * University Copyright- Copyright (c) 1982, 1986, 1988
 * The Regents of the University of California
 * All Rights Reserved
 *
 * University Acknowledgment- Portions of this document are derived from
 * software developed by the University of California, Berkeley, and its
 * contributors.
 */

#pragma ident	"@(#)vm_page.c	1.330	07/11/12 SMI"

/*
 * VM - physical page management.
 */

#include <sys/types.h>
#include <sys/t_lock.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/time.h>
#include <sys/vnode.h>
#include <sys/vm.h>
#include <sys/vtrace.h>
#include <sys/swap.h>
#include <sys/cmn_err.h>
#include <sys/tuneable.h>
#include <sys/sysmacros.h>
#include <sys/cpuvar.h>
#include <sys/callb.h>
#include <sys/debug.h>
#include <sys/tnf_probe.h>
#include <sys/condvar_impl.h>
#include <sys/mem_config.h>
#include <sys/mem_cage.h>
#include <sys/kmem.h>
#include <sys/atomic.h>
#include <sys/strlog.h>
#include <sys/mman.h>
#include <sys/ontrap.h>
#include <sys/lgrp.h>
#include <sys/vfs.h>

#include <vm/hat.h>
#include <vm/anon.h>
#include <vm/page.h>
#include <vm/seg.h>
#include <vm/pvn.h>
#include <vm/seg_kmem.h>
#include <vm/vm_dep.h>
#include <sys/vm_usage.h>
#include <fs/fs_subr.h>
#include <sys/ddi.h>
#include <sys/modctl.h>

static int nopageage = 0;

static pgcnt_t max_page_get;	/* max page_get request size in pages */
pgcnt_t total_pages = 0;	/* total number of pages (used by /proc) */

/*
 * freemem_lock protects all freemem variables:
 * availrmem. Also this lock protects the globals which track the
 * availrmem changes for accurate kernel footprint calculation.
 * See below for an explanation of these
 * globals.
 */
kmutex_t freemem_lock;
pgcnt_t availrmem;
pgcnt_t availrmem_initial;

/*
 * These globals track availrmem changes to get a more accurate
 * estimate of tke kernel size. Historically pp_kernel is used for
 * kernel size and is based on availrmem. But availrmem is adjusted for
 * locked pages in the system not just for kernel locked pages.
 * These new counters will track the pages locked through segvn and
 * by explicit user locking.
 *
 * segvn_pages_locked : This keeps track on a global basis how many pages
 * are currently locked because of I/O.
 *
 * pages_locked : How many pages are locked because of user specified
 * locking through mlock or plock.
 *
 * pages_useclaim,pages_claimed : These two variables track the
 * claim adjustments because of the protection changes on a segvn segment.
 *
 * All these globals are protected by the same lock which protects availrmem.
 */
pgcnt_t segvn_pages_locked;
pgcnt_t pages_locked;
pgcnt_t pages_useclaim;
pgcnt_t pages_claimed;


/*
 * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
 */
static kmutex_t	new_freemem_lock;
static uint_t	freemem_wait;	/* someone waiting for freemem */
static kcondvar_t freemem_cv;

/*
 * The logical page free list is maintained as two lists, the 'free'
 * and the 'cache' lists.
 * The free list contains those pages that should be reused first.
 *
 * The implementation of the lists is machine dependent.
 * page_get_freelist(), page_get_cachelist(),
 * page_list_sub(), and page_list_add()
 * form the interface to the machine dependent implementation.
 *
 * Pages with p_free set are on the cache list.
 * Pages with p_free and p_age set are on the free list,
 *
 * A page may be locked while on either list.
 */

/*
 * free list accounting stuff.
 *
 *
 * Spread out the value for the number of pages on the
 * page free and page cache lists.  If there is just one
 * value, then it must be under just one lock.
 * The lock contention and cache traffic are a real bother.
 *
 * When we acquire and then drop a single pcf lock
 * we can start in the middle of the array of pcf structures.
 * If we acquire more than one pcf lock at a time, we need to
 * start at the front to avoid deadlocking.
 *
 * pcf_count holds the number of pages in each pool.
 *
 * pcf_block is set when page_create_get_something() has asked the
 * PSM page freelist and page cachelist routines without specifying
 * a color and nothing came back.  This is used to block anything
 * else from moving pages from one list to the other while the
 * lists are searched again.  If a page is freeed while pcf_block is
 * set, then pcf_reserve is incremented.  pcgs_unblock() takes care
 * of clearning pcf_block, doing the wakeups, etc.
 */

#if NCPU <= 4
#define	PAD	2
#define	PCF_FANOUT	4
static	uint_t	pcf_mask = PCF_FANOUT - 1;
#else
#define	PAD	10
#ifdef sun4v
#define	PCF_FANOUT	32
#else
#define	PCF_FANOUT	128
#endif
static	uint_t	pcf_mask = PCF_FANOUT - 1;
#endif

struct pcf {
	kmutex_t	pcf_lock;	/* protects the structure */
	uint_t		pcf_count;	/* page count */
	uint_t		pcf_wait;	/* number of waiters */
	uint_t		pcf_block; 	/* pcgs flag to page_free() */
	uint_t		pcf_reserve; 	/* pages freed after pcf_block set */
	uint_t		pcf_fill[PAD];	/* to line up on the caches */
};

static struct	pcf	pcf[PCF_FANOUT];
#define	PCF_INDEX()	((CPU->cpu_id) & (pcf_mask))

kmutex_t	pcgs_lock;		/* serializes page_create_get_ */
kmutex_t	pcgs_cagelock;		/* serializes NOSLEEP cage allocs */
kmutex_t	pcgs_wait_lock;		/* used for delay in pcgs */
static kcondvar_t	pcgs_cv;	/* cv for delay in pcgs */

#ifdef VM_STATS

/*
 * No locks, but so what, they are only statistics.
 */

static struct page_tcnt {
	int	pc_free_cache;		/* free's into cache list */
	int	pc_free_dontneed;	/* free's with dontneed */
	int	pc_free_pageout;	/* free's from pageout */
	int	pc_free_free;		/* free's into free list */
	int	pc_free_pages;		/* free's into large page free list */
	int	pc_destroy_pages;	/* large page destroy's */
	int	pc_get_cache;		/* get's from cache list */
	int	pc_get_free;		/* get's from free list */
	int	pc_reclaim;		/* reclaim's */
	int	pc_abortfree;		/* abort's of free pages */
	int	pc_find_hit;		/* find's that find page */
	int	pc_find_miss;		/* find's that don't find page */
	int	pc_destroy_free;	/* # of free pages destroyed */
#define	PC_HASH_CNT	(4*PAGE_HASHAVELEN)
	int	pc_find_hashlen[PC_HASH_CNT+1];
	int	pc_addclaim_pages;
	int	pc_subclaim_pages;
	int	pc_free_replacement_page[2];
	int	pc_try_demote_pages[6];
	int	pc_demote_pages[2];
} pagecnt;

uint_t	hashin_count;
uint_t	hashin_not_held;
uint_t	hashin_already;

uint_t	hashout_count;
uint_t	hashout_not_held;

uint_t	page_create_count;
uint_t	page_create_not_enough;
uint_t	page_create_not_enough_again;
uint_t	page_create_zero;
uint_t	page_create_hashout;
uint_t	page_create_page_lock_failed;
uint_t	page_create_trylock_failed;
uint_t	page_create_found_one;
uint_t	page_create_hashin_failed;
uint_t	page_create_dropped_phm;

uint_t	page_create_new;
uint_t	page_create_exists;
uint_t	page_create_putbacks;
uint_t	page_create_overshoot;

uint_t	page_reclaim_zero;
uint_t	page_reclaim_zero_locked;

uint_t	page_rename_exists;
uint_t	page_rename_count;

uint_t	page_lookup_cnt[20];
uint_t	page_lookup_nowait_cnt[10];
uint_t	page_find_cnt;
uint_t	page_exists_cnt;
uint_t	page_exists_forreal_cnt;
uint_t	page_lookup_dev_cnt;
uint_t	get_cachelist_cnt;
uint_t	page_create_cnt[10];
uint_t	alloc_pages[9];
uint_t	page_exphcontg[19];
uint_t  page_create_large_cnt[10];

/*
 * Collects statistics.
 */
#define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
	uint_t	mylen = 0; \
			\
	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \
		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
			break; \
	} \
	if ((pp) != NULL) \
		pagecnt.pc_find_hit++; \
	else \
		pagecnt.pc_find_miss++; \
	if (mylen > PC_HASH_CNT) \
		mylen = PC_HASH_CNT; \
	pagecnt.pc_find_hashlen[mylen]++; \
}

#else	/* VM_STATS */

/*
 * Don't collect statistics
 */
#define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
			break; \
	} \
}

#endif	/* VM_STATS */



#ifdef DEBUG
#define	MEMSEG_SEARCH_STATS
#endif

#ifdef MEMSEG_SEARCH_STATS
struct memseg_stats {
    uint_t nsearch;
    uint_t nlastwon;
    uint_t nhashwon;
    uint_t nnotfound;
} memseg_stats;

#define	MEMSEG_STAT_INCR(v) \
	atomic_add_32(&memseg_stats.v, 1)
#else
#define	MEMSEG_STAT_INCR(x)
#endif

struct memseg *memsegs;		/* list of memory segments */


static void page_init_mem_config(void);
static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
static void page_do_hashout(page_t *);
static void page_capture_init();
int page_capture_take_action(page_t *, uint_t, void *);

static void page_demote_vp_pages(page_t *);

/*
 * vm subsystem related initialization
 */
void
vm_init(void)
{
	boolean_t callb_vm_cpr(void *, int);

	(void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
	page_init_mem_config();
	page_retire_init();
	vm_usage_init();
	page_capture_init();
}

/*
 * This function is called at startup and when memory is added or deleted.
 */
void
init_pages_pp_maximum()
{
	static pgcnt_t p_min;
	static pgcnt_t pages_pp_maximum_startup;
	static pgcnt_t avrmem_delta;
	static int init_done;
	static int user_set;	/* true if set in /etc/system */

	if (init_done == 0) {

		/* If the user specified a value, save it */
		if (pages_pp_maximum != 0) {
			user_set = 1;
			pages_pp_maximum_startup = pages_pp_maximum;
		}

		/*
		 * Setting of pages_pp_maximum is based first time
		 * on the value of availrmem just after the start-up
		 * allocations. To preserve this relationship at run
		 * time, use a delta from availrmem_initial.
		 */
		ASSERT(availrmem_initial >= availrmem);
		avrmem_delta = availrmem_initial - availrmem;

		/* The allowable floor of pages_pp_maximum */
		p_min = tune.t_minarmem + 100;

		/* Make sure we don't come through here again. */
		init_done = 1;
	}
	/*
	 * Determine pages_pp_maximum, the number of currently available
	 * pages (availrmem) that can't be `locked'. If not set by
	 * the user, we set it to 4% of the currently available memory
	 * plus 4MB.
	 * But we also insist that it be greater than tune.t_minarmem;
	 * otherwise a process could lock down a lot of memory, get swapped
	 * out, and never have enough to get swapped back in.
	 */
	if (user_set)
		pages_pp_maximum = pages_pp_maximum_startup;
	else
		pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
		    + btop(4 * 1024 * 1024);

	if (pages_pp_maximum <= p_min) {
		pages_pp_maximum = p_min;
	}
}

void
set_max_page_get(pgcnt_t target_total_pages)
{
	max_page_get = target_total_pages / 2;
}

static pgcnt_t pending_delete;

/*ARGSUSED*/
static void
page_mem_config_post_add(
	void *arg,
	pgcnt_t delta_pages)
{
	set_max_page_get(total_pages - pending_delete);
	init_pages_pp_maximum();
}

/*ARGSUSED*/
static int
page_mem_config_pre_del(
	void *arg,
	pgcnt_t delta_pages)
{
	pgcnt_t nv;

	nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
	set_max_page_get(total_pages - nv);
	return (0);
}

/*ARGSUSED*/
static void
page_mem_config_post_del(
	void *arg,
	pgcnt_t delta_pages,
	int cancelled)
{
	pgcnt_t nv;

	nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
	set_max_page_get(total_pages - nv);
	if (!cancelled)
		init_pages_pp_maximum();
}

static kphysm_setup_vector_t page_mem_config_vec = {
	KPHYSM_SETUP_VECTOR_VERSION,
	page_mem_config_post_add,
	page_mem_config_pre_del,
	page_mem_config_post_del,
};

static void
page_init_mem_config(void)
{
	int ret;

	ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
	ASSERT(ret == 0);
}

/*
 * Evenly spread out the PCF counters for large free pages
 */
static void
page_free_large_ctr(pgcnt_t npages)
{
	static struct pcf	*p = pcf;
	pgcnt_t			lump;

	freemem += npages;

	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;

	while (npages > 0) {

		ASSERT(!p->pcf_block);

		if (lump < npages) {
			p->pcf_count += (uint_t)lump;
			npages -= lump;
		} else {
			p->pcf_count += (uint_t)npages;
			npages = 0;
		}

		ASSERT(!p->pcf_wait);

		if (++p > &pcf[PCF_FANOUT - 1])
			p = pcf;
	}

	ASSERT(npages == 0);
}

/*
 * Add a physical chunk of memory to the system free lists during startup.
 * Platform specific startup() allocates the memory for the page structs.
 *
 * num	- number of page structures
 * base - page number (pfn) to be associated with the first page.
 *
 * Since we are doing this during startup (ie. single threaded), we will
 * use shortcut routines to avoid any locking overhead while putting all
 * these pages on the freelists.
 *
 * NOTE: Any changes performed to page_free(), must also be performed to
 *	 add_physmem() since this is how we initialize all page_t's at
 *	 boot time.
 */
void
add_physmem(
	page_t	*pp,
	pgcnt_t	num,
	pfn_t	pnum)
{
	page_t	*root = NULL;
	uint_t	szc = page_num_pagesizes() - 1;
	pgcnt_t	large = page_get_pagecnt(szc);
	pgcnt_t	cnt = 0;

	TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
	    "add_physmem:pp %p num %lu", pp, num);

	/*
	 * Arbitrarily limit the max page_get request
	 * to 1/2 of the page structs we have.
	 */
	total_pages += num;
	set_max_page_get(total_pages);

	PLCNT_MODIFY_MAX(pnum, (long)num);

	/*
	 * The physical space for the pages array
	 * representing ram pages has already been
	 * allocated.  Here we initialize each lock
	 * in the page structure, and put each on
	 * the free list
	 */
	for (; num; pp++, pnum++, num--) {

		/*
		 * this needs to fill in the page number
		 * and do any other arch specific initialization
		 */
		add_physmem_cb(pp, pnum);

		pp->p_lckcnt = 0;
		pp->p_cowcnt = 0;
		pp->p_slckcnt = 0;

		/*
		 * Initialize the page lock as unlocked, since nobody
		 * can see or access this page yet.
		 */
		pp->p_selock = 0;

		/*
		 * Initialize IO lock
		 */
		page_iolock_init(pp);

		/*
		 * initialize other fields in the page_t
		 */
		PP_SETFREE(pp);
		page_clr_all_props(pp);
		PP_SETAGED(pp);
		pp->p_offset = (u_offset_t)-1;
		pp->p_next = pp;
		pp->p_prev = pp;

		/*
		 * Simple case: System doesn't support large pages.
		 */
		if (szc == 0) {
			pp->p_szc = 0;
			page_free_at_startup(pp);
			continue;
		}

		/*
		 * Handle unaligned pages, we collect them up onto
		 * the root page until we have a full large page.
		 */
		if (!IS_P2ALIGNED(pnum, large)) {

			/*
			 * If not in a large page,
			 * just free as small page.
			 */
			if (root == NULL) {
				pp->p_szc = 0;
				page_free_at_startup(pp);
				continue;
			}

			/*
			 * Link a constituent page into the large page.
			 */
			pp->p_szc = szc;
			page_list_concat(&root, &pp);

			/*
			 * When large page is fully formed, free it.
			 */
			if (++cnt == large) {
				page_free_large_ctr(cnt);
				page_list_add_pages(root, PG_LIST_ISINIT);
				root = NULL;
				cnt = 0;
			}
			continue;
		}

		/*
		 * At this point we have a page number which
		 * is aligned. We assert that we aren't already
		 * in a different large page.
		 */
		ASSERT(IS_P2ALIGNED(pnum, large));
		ASSERT(root == NULL && cnt == 0);

		/*
		 * If insufficient number of pages left to form
		 * a large page, just free the small page.
		 */
		if (num < large) {
			pp->p_szc = 0;
			page_free_at_startup(pp);
			continue;
		}

		/*
		 * Otherwise start a new large page.
		 */
		pp->p_szc = szc;
		cnt++;
		root = pp;
	}
	ASSERT(root == NULL && cnt == 0);
}

/*
 * Find a page representing the specified [vp, offset].
 * If we find the page but it is intransit coming in,
 * it will have an "exclusive" lock and we wait for
 * the i/o to complete.  A page found on the free list
 * is always reclaimed and then locked.  On success, the page
 * is locked, its data is valid and it isn't on the free
 * list, while a NULL is returned if the page doesn't exist.
 */
page_t *
page_lookup(vnode_t *vp, u_offset_t off, se_t se)
{
	return (page_lookup_create(vp, off, se, NULL, NULL, 0));
}

/*
 * Find a page representing the specified [vp, offset].
 * We either return the one we found or, if passed in,
 * create one with identity of [vp, offset] of the
 * pre-allocated page. If we find existing page but it is
 * intransit coming in, it will have an "exclusive" lock
 * and we wait for the i/o to complete.  A page found on
 * the free list is always reclaimed and then locked.
 * On success, the page is locked, its data is valid and
 * it isn't on the free list, while a NULL is returned
 * if the page doesn't exist and newpp is NULL;
 */
page_t *
page_lookup_create(
	vnode_t *vp,
	u_offset_t off,
	se_t se,
	page_t *newpp,
	spgcnt_t *nrelocp,
	int flags)
{
	page_t		*pp;
	kmutex_t	*phm;
	ulong_t		index;
	uint_t		hash_locked;
	uint_t		es;

	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
	VM_STAT_ADD(page_lookup_cnt[0]);
	ASSERT(newpp ? PAGE_EXCL(newpp) : 1);

	/*
	 * Acquire the appropriate page hash lock since
	 * we have to search the hash list.  Pages that
	 * hash to this list can't change identity while
	 * this lock is held.
	 */
	hash_locked = 0;
	index = PAGE_HASH_FUNC(vp, off);
	phm = NULL;
top:
	PAGE_HASH_SEARCH(index, pp, vp, off);
	if (pp != NULL) {
		VM_STAT_ADD(page_lookup_cnt[1]);
		es = (newpp != NULL) ? 1 : 0;
		es |= flags;
		if (!hash_locked) {
			VM_STAT_ADD(page_lookup_cnt[2]);
			if (!page_try_reclaim_lock(pp, se, es)) {
				/*
				 * On a miss, acquire the phm.  Then
				 * next time, page_lock() will be called,
				 * causing a wait if the page is busy.
				 * just looping with page_trylock() would
				 * get pretty boring.
				 */
				VM_STAT_ADD(page_lookup_cnt[3]);
				phm = PAGE_HASH_MUTEX(index);
				mutex_enter(phm);
				hash_locked = 1;
				goto top;
			}
		} else {
			VM_STAT_ADD(page_lookup_cnt[4]);
			if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
				VM_STAT_ADD(page_lookup_cnt[5]);
				goto top;
			}
		}

		/*
		 * Since `pp' is locked it can not change identity now.
		 * Reconfirm we locked the correct page.
		 *
		 * Both the p_vnode and p_offset *must* be cast volatile
		 * to force a reload of their values: The PAGE_HASH_SEARCH
		 * macro will have stuffed p_vnode and p_offset into
		 * registers before calling page_trylock(); another thread,
		 * actually holding the hash lock, could have changed the
		 * page's identity in memory, but our registers would not
		 * be changed, fooling the reconfirmation.  If the hash
		 * lock was held during the search, the casting would
		 * not be needed.
		 */
		VM_STAT_ADD(page_lookup_cnt[6]);
		if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
		    ((volatile u_offset_t)(pp->p_offset) != off)) {
			VM_STAT_ADD(page_lookup_cnt[7]);
			if (hash_locked) {
				panic("page_lookup_create: lost page %p",
				    (void *)pp);
				/*NOTREACHED*/
			}
			page_unlock(pp);
			phm = PAGE_HASH_MUTEX(index);
			mutex_enter(phm);
			hash_locked = 1;
			goto top;
		}

		/*
		 * If page_trylock() was called, then pp may still be on
		 * the cachelist (can't be on the free list, it would not
		 * have been found in the search).  If it is on the
		 * cachelist it must be pulled now. To pull the page from
		 * the cachelist, it must be exclusively locked.
		 *
		 * The other big difference between page_trylock() and
		 * page_lock(), is that page_lock() will pull the
		 * page from whatever free list (the cache list in this
		 * case) the page is on.  If page_trylock() was used
		 * above, then we have to do the reclaim ourselves.
		 */
		if ((!hash_locked) && (PP_ISFREE(pp))) {
			ASSERT(PP_ISAGED(pp) == 0);
			VM_STAT_ADD(page_lookup_cnt[8]);

			/*
			 * page_relcaim will insure that we
			 * have this page exclusively
			 */

			if (!page_reclaim(pp, NULL)) {
				/*
				 * Page_reclaim dropped whatever lock
				 * we held.
				 */
				VM_STAT_ADD(page_lookup_cnt[9]);
				phm = PAGE_HASH_MUTEX(index);
				mutex_enter(phm);
				hash_locked = 1;
				goto top;
			} else if (se == SE_SHARED && newpp == NULL) {
				VM_STAT_ADD(page_lookup_cnt[10]);
				page_downgrade(pp);
			}
		}

		if (hash_locked) {
			mutex_exit(phm);
		}

		if (newpp != NULL && pp->p_szc < newpp->p_szc &&
		    PAGE_EXCL(pp) && nrelocp != NULL) {
			ASSERT(nrelocp != NULL);
			(void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
			    NULL);
			if (*nrelocp > 0) {
				VM_STAT_COND_ADD(*nrelocp == 1,
				    page_lookup_cnt[11]);
				VM_STAT_COND_ADD(*nrelocp > 1,
				    page_lookup_cnt[12]);
				pp = newpp;
				se = SE_EXCL;
			} else {
				if (se == SE_SHARED) {
					page_downgrade(pp);
				}
				VM_STAT_ADD(page_lookup_cnt[13]);
			}
		} else if (newpp != NULL && nrelocp != NULL) {
			if (PAGE_EXCL(pp) && se == SE_SHARED) {
				page_downgrade(pp);
			}
			VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
			    page_lookup_cnt[14]);
			VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
			    page_lookup_cnt[15]);
			VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
			    page_lookup_cnt[16]);
		} else if (newpp != NULL && PAGE_EXCL(pp)) {
			se = SE_EXCL;
		}
	} else if (!hash_locked) {
		VM_STAT_ADD(page_lookup_cnt[17]);
		phm = PAGE_HASH_MUTEX(index);
		mutex_enter(phm);
		hash_locked = 1;
		goto top;
	} else if (newpp != NULL) {
		/*
		 * If we have a preallocated page then
		 * insert it now and basically behave like
		 * page_create.
		 */
		VM_STAT_ADD(page_lookup_cnt[18]);
		/*
		 * Since we hold the page hash mutex and
		 * just searched for this page, page_hashin
		 * had better not fail.  If it does, that
		 * means some thread did not follow the
		 * page hash mutex rules.  Panic now and
		 * get it over with.  As usual, go down
		 * holding all the locks.
		 */
		ASSERT(MUTEX_HELD(phm));
		if (!page_hashin(newpp, vp, off, phm)) {
			ASSERT(MUTEX_HELD(phm));
			panic("page_lookup_create: hashin failed %p %p %llx %p",
			    (void *)newpp, (void *)vp, off, (void *)phm);
			/*NOTREACHED*/
		}
		ASSERT(MUTEX_HELD(phm));
		mutex_exit(phm);
		phm = NULL;
		page_set_props(newpp, P_REF);
		page_io_lock(newpp);
		pp = newpp;
		se = SE_EXCL;
	} else {
		VM_STAT_ADD(page_lookup_cnt[19]);
		mutex_exit(phm);
	}

	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);

	ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);

	return (pp);
}

/*
 * Search the hash list for the page representing the
 * specified [vp, offset] and return it locked.  Skip
 * free pages and pages that cannot be locked as requested.
 * Used while attempting to kluster pages.
 */
page_t *
page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
{
	page_t		*pp;
	kmutex_t	*phm;
	ulong_t		index;
	uint_t		locked;

	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
	VM_STAT_ADD(page_lookup_nowait_cnt[0]);

	index = PAGE_HASH_FUNC(vp, off);
	PAGE_HASH_SEARCH(index, pp, vp, off);
	locked = 0;
	if (pp == NULL) {
top:
		VM_STAT_ADD(page_lookup_nowait_cnt[1]);
		locked = 1;
		phm = PAGE_HASH_MUTEX(index);
		mutex_enter(phm);
		PAGE_HASH_SEARCH(index, pp, vp, off);
	}

	if (pp == NULL || PP_ISFREE(pp)) {
		VM_STAT_ADD(page_lookup_nowait_cnt[2]);
		pp = NULL;
	} else {
		if (!page_trylock(pp, se)) {
			VM_STAT_ADD(page_lookup_nowait_cnt[3]);
			pp = NULL;
		} else {
			VM_STAT_ADD(page_lookup_nowait_cnt[4]);
			/*
			 * See the comment in page_lookup()
			 */
			if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
			    ((u_offset_t)(pp->p_offset) != off)) {
				VM_STAT_ADD(page_lookup_nowait_cnt[5]);
				if (locked) {
					panic("page_lookup_nowait %p",
					    (void *)pp);
					/*NOTREACHED*/
				}
				page_unlock(pp);
				goto top;
			}
			if (PP_ISFREE(pp)) {
				VM_STAT_ADD(page_lookup_nowait_cnt[6]);
				page_unlock(pp);
				pp = NULL;
			}
		}
	}
	if (locked) {
		VM_STAT_ADD(page_lookup_nowait_cnt[7]);
		mutex_exit(phm);
	}

	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);

	return (pp);
}

/*
 * Search the hash list for a page with the specified [vp, off]
 * that is known to exist and is already locked.  This routine
 * is typically used by segment SOFTUNLOCK routines.
 */
page_t *
page_find(vnode_t *vp, u_offset_t off)
{
	page_t		*pp;
	kmutex_t	*phm;
	ulong_t		index;

	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
	VM_STAT_ADD(page_find_cnt);

	index = PAGE_HASH_FUNC(vp, off);
	phm = PAGE_HASH_MUTEX(index);

	mutex_enter(phm);
	PAGE_HASH_SEARCH(index, pp, vp, off);
	mutex_exit(phm);

	ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
	return (pp);
}

/*
 * Determine whether a page with the specified [vp, off]
 * currently exists in the system.  Obviously this should
 * only be considered as a hint since nothing prevents the
 * page from disappearing or appearing immediately after
 * the return from this routine. Subsequently, we don't
 * even bother to lock the list.
 */
page_t *
page_exists(vnode_t *vp, u_offset_t off)
{
	page_t	*pp;
	ulong_t		index;

	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
	VM_STAT_ADD(page_exists_cnt);

	index = PAGE_HASH_FUNC(vp, off);
	PAGE_HASH_SEARCH(index, pp, vp, off);

	return (pp);
}

/*
 * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
 * page_size(szc)) range.  if they exist and ppa is not NULL fill ppa array
 * with these pages locked SHARED. If necessary reclaim pages from
 * freelist. Return 1 if contiguous pages exist and 0 otherwise.
 *
 * If we fail to lock pages still return 1 if pages exist and contiguous.
 * But in this case return value is just a hint. ppa array won't be filled.
 * Caller should initialize ppa[0] as NULL to distinguish return value.
 *
 * Returns 0 if pages don't exist or not physically contiguous.
 *
 * This routine doesn't work for anonymous(swapfs) pages.
 */
int
page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
{
	pgcnt_t pages;
	pfn_t pfn;
	page_t *rootpp;
	pgcnt_t i;
	pgcnt_t j;
	u_offset_t save_off = off;
	ulong_t index;
	kmutex_t *phm;
	page_t *pp;
	uint_t pszc;
	int loopcnt = 0;

	ASSERT(szc != 0);
	ASSERT(vp != NULL);
	ASSERT(!IS_SWAPFSVP(vp));
	ASSERT(!VN_ISKAS(vp));

again:
	if (++loopcnt > 3) {
		VM_STAT_ADD(page_exphcontg[0]);
		return (0);
	}

	index = PAGE_HASH_FUNC(vp, off);
	phm = PAGE_HASH_MUTEX(index);

	mutex_enter(phm);
	PAGE_HASH_SEARCH(index, pp, vp, off);
	mutex_exit(phm);

	VM_STAT_ADD(page_exphcontg[1]);

	if (pp == NULL) {
		VM_STAT_ADD(page_exphcontg[2]);
		return (0);
	}

	pages = page_get_pagecnt(szc);
	rootpp = pp;
	pfn = rootpp->p_pagenum;

	if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
		VM_STAT_ADD(page_exphcontg[3]);
		if (!page_trylock(pp, SE_SHARED)) {
			VM_STAT_ADD(page_exphcontg[4]);
			return (1);
		}
		if (pp->p_szc != pszc || pp->p_vnode != vp ||
		    pp->p_offset != off) {
			VM_STAT_ADD(page_exphcontg[5]);
			page_unlock(pp);
			off = save_off;
			goto again;
		}
		/*
		 * szc was non zero and vnode and offset matched after we
		 * locked the page it means it can't become free on us.
		 */
		ASSERT(!PP_ISFREE(pp));
		if (!IS_P2ALIGNED(pfn, pages)) {
			page_unlock(pp);
			return (0);
		}
		ppa[0] = pp;
		pp++;
		off += PAGESIZE;
		pfn++;
		for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
			if (!page_trylock(pp, SE_SHARED)) {
				VM_STAT_ADD(page_exphcontg[6]);
				pp--;
				while (i-- > 0) {
					page_unlock(pp);
					pp--;
				}
				ppa[0] = NULL;
				return (1);
			}
			if (pp->p_szc != pszc) {
				VM_STAT_ADD(page_exphcontg[7]);
				page_unlock(pp);
				pp--;
				while (i-- > 0) {
					page_unlock(pp);
					pp--;
				}
				ppa[0] = NULL;
				off = save_off;
				goto again;
			}
			/*
			 * szc the same as for previous already locked pages
			 * with right identity. Since this page had correct
			 * szc after we locked it can't get freed or destroyed
			 * and therefore must have the expected identity.
			 */
			ASSERT(!PP_ISFREE(pp));
			if (pp->p_vnode != vp ||
			    pp->p_offset != off) {
				panic("page_exists_physcontig: "
				    "large page identity doesn't match");
			}
			ppa[i] = pp;
			ASSERT(pp->p_pagenum == pfn);
		}
		VM_STAT_ADD(page_exphcontg[8]);
		ppa[pages] = NULL;
		return (1);
	} else if (pszc >= szc) {
		VM_STAT_ADD(page_exphcontg[9]);
		if (!IS_P2ALIGNED(pfn, pages)) {
			return (0);
		}
		return (1);
	}

	if (!IS_P2ALIGNED(pfn, pages)) {
		VM_STAT_ADD(page_exphcontg[10]);
		return (0);
	}

	if (page_numtomemseg_nolock(pfn) !=
	    page_numtomemseg_nolock(pfn + pages - 1)) {
		VM_STAT_ADD(page_exphcontg[11]);
		return (0);
	}

	/*
	 * We loop up 4 times across pages to promote page size.
	 * We're extra cautious to promote page size atomically with respect
	 * to everybody else.  But we can probably optimize into 1 loop if
	 * this becomes an issue.
	 */

	for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
		ASSERT(pp->p_pagenum == pfn);
		if (!page_trylock(pp, SE_EXCL)) {
			VM_STAT_ADD(page_exphcontg[12]);
			break;
		}
		if (pp->p_vnode != vp ||
		    pp->p_offset != off) {
			VM_STAT_ADD(page_exphcontg[13]);
			page_unlock(pp);
			break;
		}
		if (pp->p_szc >= szc) {
			ASSERT(i == 0);
			page_unlock(pp);
			off = save_off;
			goto again;
		}
	}

	if (i != pages) {
		VM_STAT_ADD(page_exphcontg[14]);
		--pp;
		while (i-- > 0) {
			page_unlock(pp);
			--pp;
		}
		return (0);
	}

	pp = rootpp;
	for (i = 0; i < pages; i++, pp