common/vm/seg_map.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved  	*/

/*
 * Portions of this source code were derived from Berkeley 4.3 BSD
 * under license from the Regents of the University of California.
 */

#pragma ident	"@(#)seg_map.c	1.143	07/10/25 SMI"

/*
 * VM - generic vnode mapping segment.
 *
 * The segmap driver is used only by the kernel to get faster (than seg_vn)
 * mappings [lower routine overhead; more persistent cache] to random
 * vnode/offsets.  Note than the kernel may (and does) use seg_vn as well.
 */

#include <sys/types.h>
#include <sys/t_lock.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/buf.h>
#include <sys/systm.h>
#include <sys/vnode.h>
#include <sys/mman.h>
#include <sys/errno.h>
#include <sys/cred.h>
#include <sys/kmem.h>
#include <sys/vtrace.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/thread.h>
#include <sys/dumphdr.h>
#include <sys/bitmap.h>
#include <sys/lgrp.h>

#include <vm/seg_kmem.h>
#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/seg_kpm.h>
#include <vm/seg_map.h>
#include <vm/page.h>
#include <vm/pvn.h>
#include <vm/rm.h>

/*
 * Private seg op routines.
 */
static void	segmap_free(struct seg *seg);
faultcode_t segmap_fault(struct hat *hat, struct seg *seg, caddr_t addr,
			size_t len, enum fault_type type, enum seg_rw rw);
static faultcode_t segmap_faulta(struct seg *seg, caddr_t addr);
static int	segmap_checkprot(struct seg *seg, caddr_t addr, size_t len,
			uint_t prot);
static int	segmap_kluster(struct seg *seg, caddr_t addr, ssize_t);
static int	segmap_getprot(struct seg *seg, caddr_t addr, size_t len,
			uint_t *protv);
static u_offset_t	segmap_getoffset(struct seg *seg, caddr_t addr);
static int	segmap_gettype(struct seg *seg, caddr_t addr);
static int	segmap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
static void	segmap_dump(struct seg *seg);
static int	segmap_pagelock(struct seg *seg, caddr_t addr, size_t len,
			struct page ***ppp, enum lock_type type,
			enum seg_rw rw);
static void	segmap_badop(void);
static int	segmap_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
static lgrp_mem_policy_info_t	*segmap_getpolicy(struct seg *seg,
    caddr_t addr);
static int	segmap_capable(struct seg *seg, segcapability_t capability);

/* segkpm support */
static caddr_t	segmap_pagecreate_kpm(struct seg *, vnode_t *, u_offset_t,
			struct smap *, enum seg_rw);
struct smap	*get_smap_kpm(caddr_t, page_t **);

#define	SEGMAP_BADOP(t)	(t(*)())segmap_badop

static struct seg_ops segmap_ops = {
	SEGMAP_BADOP(int),	/* dup */
	SEGMAP_BADOP(int),	/* unmap */
	segmap_free,
	segmap_fault,
	segmap_faulta,
	SEGMAP_BADOP(int),	/* setprot */
	segmap_checkprot,
	segmap_kluster,
	SEGMAP_BADOP(size_t),	/* swapout */
	SEGMAP_BADOP(int),	/* sync */
	SEGMAP_BADOP(size_t),	/* incore */
	SEGMAP_BADOP(int),	/* lockop */
	segmap_getprot,
	segmap_getoffset,
	segmap_gettype,
	segmap_getvp,
	SEGMAP_BADOP(int),	/* advise */
	segmap_dump,
	segmap_pagelock,	/* pagelock */
	SEGMAP_BADOP(int),	/* setpgsz */
	segmap_getmemid,	/* getmemid */
	segmap_getpolicy,	/* getpolicy */
	segmap_capable,		/* capable */
};

/*
 * Private segmap routines.
 */
static void	segmap_unlock(struct hat *hat, struct seg *seg, caddr_t addr,
			size_t len, enum seg_rw rw, struct smap *smp);
static void	segmap_smapadd(struct smap *smp);
static struct smap *segmap_hashin(struct smap *smp, struct vnode *vp,
			u_offset_t off, int hashid);
static void	segmap_hashout(struct smap *smp);


/*
 * Statistics for segmap operations.
 *
 * No explicit locking to protect these stats.
 */
struct segmapcnt segmapcnt = {
	{ "fault",		KSTAT_DATA_ULONG },
	{ "faulta",		KSTAT_DATA_ULONG },
	{ "getmap",		KSTAT_DATA_ULONG },
	{ "get_use",		KSTAT_DATA_ULONG },
	{ "get_reclaim",	KSTAT_DATA_ULONG },
	{ "get_reuse",		KSTAT_DATA_ULONG },
	{ "get_unused",		KSTAT_DATA_ULONG },
	{ "get_nofree",		KSTAT_DATA_ULONG },
	{ "rel_async",		KSTAT_DATA_ULONG },
	{ "rel_write",		KSTAT_DATA_ULONG },
	{ "rel_free",		KSTAT_DATA_ULONG },
	{ "rel_abort",		KSTAT_DATA_ULONG },
	{ "rel_dontneed",	KSTAT_DATA_ULONG },
	{ "release",		KSTAT_DATA_ULONG },
	{ "pagecreate",		KSTAT_DATA_ULONG },
	{ "free_notfree",	KSTAT_DATA_ULONG },
	{ "free_dirty",		KSTAT_DATA_ULONG },
	{ "free",		KSTAT_DATA_ULONG },
	{ "stolen",		KSTAT_DATA_ULONG },
	{ "get_nomtx",		KSTAT_DATA_ULONG }
};

kstat_named_t *segmapcnt_ptr = (kstat_named_t *)&segmapcnt;
uint_t segmapcnt_ndata = sizeof (segmapcnt) / sizeof (kstat_named_t);

/*
 * Return number of map pages in segment.
 */
#define	MAP_PAGES(seg)		((seg)->s_size >> MAXBSHIFT)

/*
 * Translate addr into smap number within segment.
 */
#define	MAP_PAGE(seg, addr)  (((addr) - (seg)->s_base) >> MAXBSHIFT)

/*
 * Translate addr in seg into struct smap pointer.
 */
#define	GET_SMAP(seg, addr)	\
	&(((struct segmap_data *)((seg)->s_data))->smd_sm[MAP_PAGE(seg, addr)])

/*
 * Bit in map (16 bit bitmap).
 */
#define	SMAP_BIT_MASK(bitindex)	(1 << ((bitindex) & 0xf))

static int smd_colormsk = 0;
static int smd_ncolor = 0;
static int smd_nfree = 0;
static int smd_freemsk = 0;
#ifdef DEBUG
static int *colors_used;
#endif
static struct smap *smd_smap;
static struct smaphash *smd_hash;
#ifdef SEGMAP_HASHSTATS
static unsigned int *smd_hash_len;
#endif
static struct smfree *smd_free;
static ulong_t smd_hashmsk = 0;

#define	SEGMAP_MAXCOLOR		2
#define	SEGMAP_CACHE_PAD	64

union segmap_cpu {
	struct {
		uint32_t	scpu_free_ndx[SEGMAP_MAXCOLOR];
		struct smap	*scpu_last_smap;
		ulong_t		scpu_getmap;
		ulong_t		scpu_release;
		ulong_t		scpu_get_reclaim;
		ulong_t		scpu_fault;
		ulong_t		scpu_pagecreate;
		ulong_t		scpu_get_reuse;
	} scpu;
	char	scpu_pad[SEGMAP_CACHE_PAD];
};
static union segmap_cpu *smd_cpu;

/*
 * There are three locks in seg_map:
 *	- per freelist mutexes
 *	- per hashchain mutexes
 *	- per smap mutexes
 *
 * The lock ordering is to get the smap mutex to lock down the slot
 * first then the hash lock (for hash in/out (vp, off) list) or the
 * freelist lock to put the slot back on the free list.
 *
 * The hash search is done by only holding the hashchain lock, when a wanted
 * slot is found, we drop the hashchain lock then lock the slot so there
 * is no overlapping of hashchain and smap locks. After the slot is
 * locked, we verify again if the slot is still what we are looking
 * for.
 *
 * Allocation of a free slot is done by holding the freelist lock,
 * then locking the smap slot at the head of the freelist. This is
 * in reversed lock order so mutex_tryenter() is used.
 *
 * The smap lock protects all fields in smap structure except for
 * the link fields for hash/free lists which are protected by
 * hashchain and freelist locks.
 */

#define	SHASHMTX(hashid)	(&smd_hash[hashid].sh_mtx)

#define	SMP2SMF(smp)		(&smd_free[(smp - smd_smap) & smd_freemsk])
#define	SMP2SMF_NDX(smp)	(ushort_t)((smp - smd_smap) & smd_freemsk)

#define	SMAPMTX(smp) (&smp->sm_mtx)

#define	SMAP_HASHFUNC(vp, off, hashid) \
	{ \
	hashid = ((((uintptr_t)(vp) >> 6) + ((uintptr_t)(vp) >> 3) + \
		((off) >> MAXBSHIFT)) & smd_hashmsk); \
	}

/*
 * The most frequently updated kstat counters are kept in the
 * per cpu array to avoid hot cache blocks. The update function
 * sums the cpu local counters to update the global counters.
 */

/* ARGSUSED */
int
segmap_kstat_update(kstat_t *ksp, int rw)
{
	int i;
	ulong_t	getmap, release, get_reclaim;
	ulong_t	fault, pagecreate, get_reuse;

	if (rw == KSTAT_WRITE)
		return (EACCES);
	getmap = release = get_reclaim = (ulong_t)0;
	fault = pagecreate = get_reuse = (ulong_t)0;
	for (i = 0; i < max_ncpus; i++) {
		getmap += smd_cpu[i].scpu.scpu_getmap;
		release  += smd_cpu[i].scpu.scpu_release;
		get_reclaim += smd_cpu[i].scpu.scpu_get_reclaim;
		fault  += smd_cpu[i].scpu.scpu_fault;
		pagecreate  += smd_cpu[i].scpu.scpu_pagecreate;
		get_reuse += smd_cpu[i].scpu.scpu_get_reuse;
	}
	segmapcnt.smp_getmap.value.ul = getmap;
	segmapcnt.smp_release.value.ul = release;
	segmapcnt.smp_get_reclaim.value.ul = get_reclaim;
	segmapcnt.smp_fault.value.ul = fault;
	segmapcnt.smp_pagecreate.value.ul = pagecreate;
	segmapcnt.smp_get_reuse.value.ul = get_reuse;
	return (0);
}

int
segmap_create(struct seg *seg, void *argsp)
{
	struct segmap_data *smd;
	struct smap *smp;
	struct smfree *sm;
	struct segmap_crargs *a = (struct segmap_crargs *)argsp;
	struct smaphash *shashp;
	union segmap_cpu *scpu;
	long i, npages;
	size_t hashsz;
	uint_t nfreelist;
	extern void prefetch_smap_w(void *);
	extern int max_ncpus;

	ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));

	if (((uintptr_t)seg->s_base | seg->s_size) & MAXBOFFSET) {
		panic("segkmap not MAXBSIZE aligned");
		/*NOTREACHED*/
	}

	smd = kmem_zalloc(sizeof (struct segmap_data), KM_SLEEP);

	seg->s_data = (void *)smd;
	seg->s_ops = &segmap_ops;
	smd->smd_prot = a->prot;

	/*
	 * Scale the number of smap freelists to be
	 * proportional to max_ncpus * number of virtual colors.
	 * The caller can over-ride this scaling by providing
	 * a non-zero a->nfreelist argument.
	 */
	nfreelist = a->nfreelist;
	if (nfreelist == 0)
		nfreelist = max_ncpus;
	else if (nfreelist < 0 || nfreelist > 4 * max_ncpus) {
		cmn_err(CE_WARN, "segmap_create: nfreelist out of range "
		"%d, using %d", nfreelist, max_ncpus);
		nfreelist = max_ncpus;
	}
	if (nfreelist & (nfreelist - 1)) {
		/* round up nfreelist to the next power of two. */
		nfreelist = 1 << (highbit(nfreelist));
	}

	/*
	 * Get the number of virtual colors - must be a power of 2.
	 */
	if (a->shmsize)
		smd_ncolor = a->shmsize >> MAXBSHIFT;
	else
		smd_ncolor = 1;
	ASSERT((smd_ncolor & (smd_ncolor - 1)) == 0);
	ASSERT(smd_ncolor <= SEGMAP_MAXCOLOR);
	smd_colormsk = smd_ncolor - 1;
	smd->smd_nfree = smd_nfree = smd_ncolor * nfreelist;
	smd_freemsk = smd_nfree - 1;

	/*
	 * Allocate and initialize the freelist headers.
	 * Note that sm_freeq[1] starts out as the release queue. This
	 * is known when the smap structures are initialized below.
	 */
	smd_free = smd->smd_free =
	    kmem_zalloc(smd_nfree * sizeof (struct smfree), KM_SLEEP);
	for (i = 0; i < smd_nfree; i++) {
		sm = &smd->smd_free[i];
		mutex_init(&sm->sm_freeq[0].smq_mtx, NULL, MUTEX_DEFAULT, NULL);
		mutex_init(&sm->sm_freeq[1].smq_mtx, NULL, MUTEX_DEFAULT, NULL);
		sm->sm_allocq = &sm->sm_freeq[0];
		sm->sm_releq = &sm->sm_freeq[1];
	}

	/*
	 * Allocate and initialize the smap hash chain headers.
	 * Compute hash size rounding down to the next power of two.
	 */
	npages = MAP_PAGES(seg);
	smd->smd_npages = npages;
	hashsz = npages / SMAP_HASHAVELEN;
	hashsz = 1 << (highbit(hashsz)-1);
	smd_hashmsk = hashsz - 1;
	smd_hash = smd->smd_hash =
	    kmem_alloc(hashsz * sizeof (struct smaphash), KM_SLEEP);
#ifdef SEGMAP_HASHSTATS
	smd_hash_len =
	    kmem_zalloc(hashsz * sizeof (unsigned int), KM_SLEEP);
#endif
	for (i = 0, shashp = smd_hash; i < hashsz; i++, shashp++) {
		shashp->sh_hash_list = NULL;
		mutex_init(&shashp->sh_mtx, NULL, MUTEX_DEFAULT, NULL);
	}

	/*
	 * Allocate and initialize the smap structures.
	 * Link all slots onto the appropriate freelist.
	 * The smap array is large enough to affect boot time
	 * on large systems, so use memory prefetching and only
	 * go through the array 1 time. Inline a optimized version
	 * of segmap_smapadd to add structures to freelists with
	 * knowledge that no locks are needed here.
	 */
	smd_smap = smd->smd_sm =
		kmem_alloc(sizeof (struct smap) * npages, KM_SLEEP);

	for (smp = &smd->smd_sm[MAP_PAGES(seg) - 1];
	    smp >= smd->smd_sm; smp--) {
		struct smap *smpfreelist;
		struct sm_freeq *releq;

		prefetch_smap_w((char *)smp);

		smp->sm_vp = NULL;
		smp->sm_hash = NULL;
		smp->sm_off = 0;
		smp->sm_bitmap = 0;
		smp->sm_refcnt = 0;
		mutex_init(&smp->sm_mtx, NULL, MUTEX_DEFAULT, NULL);
		smp->sm_free_ndx = SMP2SMF_NDX(smp);

		sm = SMP2SMF(smp);
		releq = sm->sm_releq;

		smpfreelist = releq->smq_free;
		if (smpfreelist == 0) {
			releq->smq_free = smp->sm_next = smp->sm_prev = smp;
		} else {
			smp->sm_next = smpfreelist;
			smp->sm_prev = smpfreelist->sm_prev;
			smpfreelist->sm_prev = smp;
			smp->sm_prev->sm_next = smp;
			releq->smq_free = smp->sm_next;
		}

		/*
		 * sm_flag = 0 (no SM_QNDX_ZERO) implies smap on sm_freeq[1]
		 */
		smp->sm_flags = 0;

#ifdef	SEGKPM_SUPPORT
		/*
		 * Due to the fragile prefetch loop no
		 * separate function is used here.
		 */
		smp->sm_kpme_next = NULL;
		smp->sm_kpme_prev = NULL;
		smp->sm_kpme_page = NULL;
#endif
	}

	/*
	 * Allocate the per color indices that distribute allocation
	 * requests over the free lists. Each cpu will have a private
	 * rotor index to spread the allocations even across the available
	 * smap freelists. Init the scpu_last_smap field to the first
	 * smap element so there is no need to check for NULL.
	 */
	smd_cpu =
		kmem_zalloc(sizeof (union segmap_cpu) * max_ncpus, KM_SLEEP);
	for (i = 0, scpu = smd_cpu; i < max_ncpus; i++, scpu++) {
		int j;
		for (j = 0; j < smd_ncolor; j++)
			scpu->scpu.scpu_free_ndx[j] = j;
		scpu->scpu.scpu_last_smap = smd_smap;
	}

	if (vpm_enable) {
		vpm_init();
	}

#ifdef DEBUG
	/*
	 * Keep track of which colors are used more often.
	 */
	colors_used = kmem_zalloc(smd_nfree * sizeof (int), KM_SLEEP);
#endif /* DEBUG */

	return (0);
}

static void
segmap_free(seg)
	struct seg *seg;
{
	ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
}

/*
 * Do a F_SOFTUNLOCK call over the range requested.
 * The range must have already been F_SOFTLOCK'ed.
 */
static void
segmap_unlock(
	struct hat *hat,
	struct seg *seg,
	caddr_t addr,
	size_t len,
	enum seg_rw rw,
	struct smap *smp)
{
	page_t *pp;
	caddr_t adr;
	u_offset_t off;
	struct vnode *vp;
	kmutex_t *smtx;

	ASSERT(smp->sm_refcnt > 0);

#ifdef lint
	seg = seg;
#endif

	if (segmap_kpm && IS_KPM_ADDR(addr)) {

		/*
		 * We're called only from segmap_fault and this was a
		 * NOP in case of a kpm based smap, so dangerous things
		 * must have happened in the meantime. Pages are prefaulted
		 * and locked in segmap_getmapflt and they will not be
		 * unlocked until segmap_release.
		 */
		panic("segmap_unlock: called with kpm addr %p", (void *)addr);
		/*NOTREACHED*/
	}

	vp = smp->sm_vp;
	off = smp->sm_off + (u_offset_t)((uintptr_t)addr & MAXBOFFSET);

	hat_unlock(hat, addr, P2ROUNDUP(len, PAGESIZE));
	for (adr = addr; adr < addr + len; adr += PAGESIZE, off += PAGESIZE) {
		ushort_t bitmask;

		/*
		 * Use page_find() instead of page_lookup() to
		 * find the page since we know that it has
		 * "shared" lock.
		 */
		pp = page_find(vp, off);
		if (pp == NULL) {
			panic("segmap_unlock: page not found");
			/*NOTREACHED*/
		}

		if (rw == S_WRITE) {
			hat_setrefmod(pp);
		} else if (rw != S_OTHER) {
			TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
				"segmap_fault:pp %p vp %p offset %llx",
				pp, vp, off);
			hat_setref(pp);
		}

		/*
		 * Clear bitmap, if the bit corresponding to "off" is set,
		 * since the page and translation are being unlocked.
		 */
		bitmask = SMAP_BIT_MASK((off - smp->sm_off) >> PAGESHIFT);

		/*
		 * Large Files: Following assertion is to verify
		 * the correctness of the cast to (int) above.
		 */
		ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
		smtx = SMAPMTX(smp);
		mutex_enter(smtx);
		if (smp->sm_bitmap & bitmask) {
			smp->sm_bitmap &= ~bitmask;
		}
		mutex_exit(smtx);

		page_unlock(pp);
	}
}

#define	MAXPPB	(MAXBSIZE/4096)	/* assumes minimum page size of 4k */

/*
 * This routine is called via a machine specific fault handling
 * routine.  It is also called by software routines wishing to
 * lock or unlock a range of addresses.
 *
 * Note that this routine expects a page-aligned "addr".
 */
faultcode_t
segmap_fault(
	struct hat *hat,
	struct seg *seg,
	caddr_t addr,
	size_t len,
	enum fault_type type,
	enum seg_rw rw)
{
	struct segmap_data *smd = (struct segmap_data *)seg->s_data;
	struct smap *smp;
	page_t *pp, **ppp;
	struct vnode *vp;
	u_offset_t off;
	page_t *pl[MAXPPB + 1];
	uint_t prot;
	u_offset_t addroff;
	caddr_t adr;
	int err;
	u_offset_t sm_off;
	int hat_flag;

	if (segmap_kpm && IS_KPM_ADDR(addr)) {
		int newpage;
		kmutex_t *smtx;

		/*
		 * Pages are successfully prefaulted and locked in
		 * segmap_getmapflt and can't be unlocked until
		 * segmap_release. No hat mappings have to be locked
		 * and they also can't be unlocked as long as the
		 * caller owns an active kpm addr.
		 */
#ifndef DEBUG
		if (type != F_SOFTUNLOCK)
			return (0);
#endif

		if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
			panic("segmap_fault: smap not found "
			    "for addr %p", (void *)addr);
			/*NOTREACHED*/
		}

		smtx = SMAPMTX(smp);
#ifdef	DEBUG
		newpage = smp->sm_flags & SM_KPM_NEWPAGE;
		if (newpage) {
			cmn_err(CE_WARN, "segmap_fault: newpage? smp %p",
				(void *)smp);
		}

		if (type != F_SOFTUNLOCK) {
			mutex_exit(smtx);
			return (0);
		}
#endif
		mutex_exit(smtx);
		vp = smp->sm_vp;
		sm_off = smp->sm_off;

		if (vp == NULL)
			return (FC_MAKE_ERR(EIO));

		ASSERT(smp->sm_refcnt > 0);

		addroff = (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
		if (addroff + len > MAXBSIZE)
			panic("segmap_fault: endaddr %p exceeds MAXBSIZE chunk",
			    (void *)(addr + len));

		off = sm_off + addroff;

		pp = page_find(vp, off);

		if (pp == NULL)
			panic("segmap_fault: softunlock page not found");

		/*
		 * Set ref bit also here in case of S_OTHER to avoid the
		 * overhead of supporting other cases than F_SOFTUNLOCK
		 * with segkpm. We can do this because the underlying
		 * pages are locked anyway.
		 */
		if (rw == S_WRITE) {
			hat_setrefmod(pp);
		} else {
			TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
				"segmap_fault:pp %p vp %p offset %llx",
				pp, vp, off);
			hat_setref(pp);
		}

		return (0);
	}

	smd_cpu[CPU->cpu_seqid].scpu.scpu_fault++;
	smp = GET_SMAP(seg, addr);
	vp = smp->sm_vp;
	sm_off = smp->sm_off;

	if (vp == NULL)
		return (FC_MAKE_ERR(EIO));

	ASSERT(smp->sm_refcnt > 0);

	addroff = (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
	if (addroff + len > MAXBSIZE) {
		panic("segmap_fault: endaddr %p "
		    "exceeds MAXBSIZE chunk", (void *)(addr + len));
		/*NOTREACHED*/
	}
	off = sm_off + addroff;

	/*
	 * First handle the easy stuff
	 */
	if (type == F_SOFTUNLOCK) {
		segmap_unlock(hat, seg, addr, len, rw, smp);
		return (0);
	}

	TRACE_3(TR_FAC_VM, TR_SEGMAP_GETPAGE,
		"segmap_getpage:seg %p addr %p vp %p", seg, addr, vp);
	err = VOP_GETPAGE(vp, (offset_t)off, len, &prot, pl, MAXBSIZE,
	    seg, addr, rw, CRED(), NULL);

	if (err)
		return (FC_MAKE_ERR(err));

	prot &= smd->smd_prot;

	/*
	 * Handle all pages returned in the pl[] array.
	 * This loop is coded on the assumption that if
	 * there was no error from the VOP_GETPAGE routine,
	 * that the page list returned will contain all the
	 * needed pages for the vp from [off..off + len].
	 */
	ppp = pl;
	while ((pp = *ppp++) != NULL) {
		u_offset_t poff;
		ASSERT(pp->p_vnode == vp);
		hat_flag = HAT_LOAD;

		/*
		 * Verify that the pages returned are within the range
		 * of this segmap region.  Note that it is theoretically
		 * possible for pages outside this range to be returned,
		 * but it is not very likely.  If we cannot use the
		 * page here, just release it and go on to the next one.
		 */
		if (pp->p_offset < sm_off ||
		    pp->p_offset >= sm_off + MAXBSIZE) {
			(void) page_release(pp, 1);
			continue;
		}

		ASSERT(hat == kas.a_hat);
		poff = pp->p_offset;
		adr = addr + (poff - off);
		if (adr >= addr && adr < addr + len) {
			hat_setref(pp);
			TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
			    "segmap_fault:pp %p vp %p offset %llx",
			    pp, vp, poff);
			if (type == F_SOFTLOCK)
				hat_flag = HAT_LOAD_LOCK;
		}

		/*
		 * Deal with VMODSORT pages here. If we know this is a write
		 * do the setmod now and allow write protection.
		 * As long as it's modified or not S_OTHER, remove write
		 * protection. With S_OTHER it's up to the FS to deal with this.
		 */
		if (IS_VMODSORT(vp)) {
			if (rw == S_WRITE)
				hat_setmod(pp);
			else if (rw != S_OTHER && !hat_ismod(pp))
				prot &= ~PROT_WRITE;
		}

		hat_memload(hat, adr, pp, prot, hat_flag);
		if (hat_flag != HAT_LOAD_LOCK)
			page_unlock(pp);
	}
	return (0);
}

/*
 * This routine is used to start I/O on pages asynchronously.
 */
static faultcode_t
segmap_faulta(struct seg *seg, caddr_t addr)
{
	struct smap *smp;
	struct vnode *vp;
	u_offset_t off;
	int err;

	if (segmap_kpm && IS_KPM_ADDR(addr)) {
		int	newpage;
		kmutex_t *smtx;

		/*
		 * Pages are successfully prefaulted and locked in
		 * segmap_getmapflt and can't be unlocked until
		 * segmap_release. No hat mappings have to be locked
		 * and they also can't be unlocked as long as the
		 * caller owns an active kpm addr.
		 */
#ifdef	DEBUG
		if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
			panic("segmap_faulta: smap not found "
			    "for addr %p", (void *)addr);
			/*NOTREACHED*/
		}

		smtx = SMAPMTX(smp);
		newpage = smp->sm_flags & SM_KPM_NEWPAGE;
		mutex_exit(smtx);
		if (newpage)
			cmn_err(CE_WARN, "segmap_faulta: newpage? smp %p",
			    (void *)smp);
#endif
		return (0);
	}

	segmapcnt.smp_faulta.value.ul++;
	smp = GET_SMAP(seg, addr);

	ASSERT(smp->sm_refcnt > 0);

	vp = smp->sm_vp;
	off = smp->sm_off;

	if (vp == NULL) {
		cmn_err(CE_WARN, "segmap_faulta - no vp");
		return (FC_MAKE_ERR(EIO));
	}

	TRACE_3(TR_FAC_VM, TR_SEGMAP_GETPAGE,
		"segmap_getpage:seg %p addr %p vp %p", seg, addr, vp);

	err = VOP_GETPAGE(vp, (offset_t)(off + ((offset_t)((uintptr_t)addr
	    & MAXBOFFSET))), PAGESIZE, (uint_t *)NULL, (page_t **)NULL, 0,
	    seg, addr, S_READ, CRED(), NULL);

	if (err)
		return (FC_MAKE_ERR(err));
	return (0);
}

/*ARGSUSED*/
static int
segmap_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
{
	struct segmap_data *smd = (struct segmap_data *)seg->s_data;

	ASSERT(seg->s_as && RW_LOCK_HELD(&seg->s_as->a_lock));

	/*
	 * Need not acquire the segment lock since
	 * "smd_prot" is a read-only field.
	 */
	return (((smd->smd_prot & prot) != prot) ? EACCES : 0);
}

static int
segmap_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
{
	struct segmap_data *smd = (struct segmap_data *)seg->s_data;
	size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));

	if (pgno != 0) {
		do
			protv[--pgno] = smd->smd_prot;
		while (pgno != 0);
	}
	return (0);
}

static u_offset_t
segmap_getoffset(struct seg *seg, caddr_t addr)
{
	struct segmap_data *smd = (struct segmap_data *)seg->s_data;

	ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));

	return ((u_offset_t)smd->smd_sm->sm_off + (addr - seg->s_base));
}

/*ARGSUSED*/
static int
segmap_gettype(struct seg *seg, caddr_t addr)
{
	ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));

	return (MAP_SHARED);
}

/*ARGSUSED*/
static int
segmap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
{
	struct segmap_data *smd = (struct segmap_data *)seg->s_data;

	ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));

	/* XXX - This doesn't make any sense */
	*vpp = smd->smd_sm->sm_vp;
	return (0);
}

/*
 * Check to see if it makes sense to do kluster/read ahead to
 * addr + delta relative to the mapping at addr.  We assume here
 * that delta is a signed PAGESIZE'd multiple (which can be negative).
 *
 * For segmap we always "approve" of this action from our standpoint.
 */
/*ARGSUSED*/
static int
segmap_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
{
	return (0);
}

static void
segmap_badop()
{
	panic("segmap_badop");
	/*NOTREACHED*/
}

/*
 * Special private segmap operations
 */

/*
 * Add smap to the appropriate free list.
 */
static void
segmap_smapadd(struct smap *smp)
{
	struct smfree *sm;
	struct smap *smpfreelist;
	struct sm_freeq *releq;

	ASSERT(MUTEX_HELD(SMAPMTX(smp)));

	if (smp->sm_refcnt != 0) {
		panic("segmap_smapadd");
		/*NOTREACHED*/
	}

	sm = &smd_free[smp->sm_free_ndx];
	/*
	 * Add to the tail of the release queue
	 * Note that sm_releq and sm_allocq could toggle
	 * before we get the lock. This does not affect
	 * correctness as the 2 queues are only maintained
	 * to reduce lock pressure.
	 */
	releq = sm->sm_releq;
	if (releq == &sm->sm_freeq[0])
		smp->sm_flags |= SM_QNDX_ZERO;
	else
		smp->sm_flags &= ~SM_QNDX_ZERO;
	mutex_enter(&releq->smq_mtx);
	smpfreelist = releq->smq_free;
	if (smpfreelist == 0) {
		int want;

		releq->smq_free = smp->sm_next = smp->sm_prev = smp;
		/*
		 * Both queue mutexes held to set sm_want;
		 * snapshot the value before dropping releq mutex.
		 * If sm_want appears after the releq mutex is dropped,
		 * then the smap just freed is already gone.
		 */
		want = sm->sm_want;
		mutex_exit(&releq->smq_mtx);
		/*
		 * See if there was a waiter before dropping the releq mutex
		 * then recheck after obtaining sm_freeq[0] mutex as
		 * the another thread may have already signaled.
		 */
		if (want) {
			mutex_enter(&sm->sm_freeq[0].smq_mtx);
			if (sm->sm_want)
				cv_signal(&sm->sm_free_cv);
			mutex_exit(&sm->sm_freeq[0].smq_mtx);
		}
	} else {
		smp->sm_next = smpfreelist;
		smp->sm_prev = smpfreelist->sm_prev;
		smpfreelist->sm_prev = smp;
		smp->sm_prev->sm_next = smp;
		mutex_exit(&releq->smq_mtx);
	}
}


static struct smap *
segmap_hashin(struct smap *smp, struct vnode *vp, u_offset_t off, int hashid)
{
	struct smap **hpp;
	struct smap *tmp;
	kmutex_t *hmtx;

	ASSERT(MUTEX_HELD(SMAPMTX(smp)));
	ASSERT(smp->sm_vp == NULL);
	ASSERT(smp->sm_hash == NULL);
	ASSERT(smp->sm_prev == NULL);
	ASSERT(smp->sm_next == NULL);
	ASSERT(hashid >= 0 && hashid <= smd_hashmsk);

	hmtx = SHASHMTX(hashid);

	mutex_enter(hmtx);
	/*
	 * First we need to verify that no one has created a smp
	 * with (vp,off) as its tag before we us.
	 */
	for (tmp = smd_hash[hashid].sh_hash_list;
	    tmp != NULL; tmp = tmp->sm_hash)
		if (tmp->sm_vp == vp && tmp->sm_off == off)
			break;

	if (tmp == NULL) {
		/*
		 * No one created one yet.
		 *
		 * Funniness here - we don't increment the ref count on the
		 * vnode * even though we have another pointer to it here.
		 * The reason for this is that we don't want the fact that
		 * a seg_map entry somewhere refers to a vnode to prevent the
		 * vnode * itself from going away.  This is because this
		 * reference to the vnode is a "soft one".  In the case where
		 * a mapping is being used by a rdwr [or directory routine?]
		 * there already has to be a non-zero ref count on the vnode.
		 * In the case where the vp has been freed and the the smap
		 * structure is on the free list, there are no pages in memory
		 * that can refer to the vnode.  Thus even if we reuse the same
		 * vnode/smap structure for a vnode which has the same
		 * address but represents a different object, we are ok.
		 */
		smp->sm_vp = vp;
		smp->sm_off = off;

		hpp = &smd_hash[hashid].sh_hash_list;
		smp->sm_hash = *hpp;
		*hpp = smp;
#ifdef SEGMAP_HASHSTATS
		smd_hash_len[hashid]++;
#endif
	}
	mutex_exit(hmtx);