xref: /netvirt/usr/src/uts/common/vm/seg_spt.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"@(#)seg_spt.c	1.106	07/10/10 SMI"

#include <sys/param.h>
#include <sys/user.h>
#include <sys/mman.h>
#include <sys/kmem.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/tuneable.h>
#include <vm/hat.h>
#include <vm/seg.h>
#include <vm/as.h>
#include <vm/anon.h>
#include <vm/page.h>
#include <sys/buf.h>
#include <sys/swap.h>
#include <sys/atomic.h>
#include <vm/seg_spt.h>
#include <sys/debug.h>
#include <sys/vtrace.h>
#include <sys/shm.h>
#include <sys/shm_impl.h>
#include <sys/lgrp.h>
#include <sys/vmsystm.h>
#include <sys/policy.h>
#include <sys/project.h>
#include <sys/tnf_probe.h>
#include <sys/zone.h>

#define	SEGSPTADDR	(caddr_t)0x0

/*
 * # pages used for spt
 */
size_t	spt_used;

/*
 * segspt_minfree is the memory left for system after ISM
 * locked its pages; it is set up to 5% of availrmem in
 * sptcreate when ISM is created.  ISM should not use more
 * than ~90% of availrmem; if it does, then the performance
 * of the system may decrease. Machines with large memories may
 * be able to use up more memory for ISM so we set the default
 * segspt_minfree to 5% (which gives ISM max 95% of availrmem.
 * If somebody wants even more memory for ISM (risking hanging
 * the system) they can patch the segspt_minfree to smaller number.
 */
pgcnt_t segspt_minfree = 0;

static int segspt_create(struct seg *seg, caddr_t argsp);
static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
static void segspt_free(struct seg *seg);
static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);

static void
segspt_badop()
{
	panic("segspt_badop called");
	/*NOTREACHED*/
}

#define	SEGSPT_BADOP(t)	(t(*)())segspt_badop

struct seg_ops segspt_ops = {
	SEGSPT_BADOP(int),		/* dup */
	segspt_unmap,
	segspt_free,
	SEGSPT_BADOP(int),		/* fault */
	SEGSPT_BADOP(faultcode_t),	/* faulta */
	SEGSPT_BADOP(int),		/* setprot */
	SEGSPT_BADOP(int),		/* checkprot */
	SEGSPT_BADOP(int),		/* kluster */
	SEGSPT_BADOP(size_t),		/* swapout */
	SEGSPT_BADOP(int),		/* sync */
	SEGSPT_BADOP(size_t),		/* incore */
	SEGSPT_BADOP(int),		/* lockop */
	SEGSPT_BADOP(int),		/* getprot */
	SEGSPT_BADOP(u_offset_t), 	/* getoffset */
	SEGSPT_BADOP(int),		/* gettype */
	SEGSPT_BADOP(int),		/* getvp */
	SEGSPT_BADOP(int),		/* advise */
	SEGSPT_BADOP(void),		/* dump */
	SEGSPT_BADOP(int),		/* pagelock */
	SEGSPT_BADOP(int),		/* setpgsz */
	SEGSPT_BADOP(int),		/* getmemid */
	segspt_getpolicy,		/* getpolicy */
	SEGSPT_BADOP(int),		/* capable */
};

static int segspt_shmdup(struct seg *seg, struct seg *newseg);
static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
static void segspt_shmfree(struct seg *seg);
static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
		caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr,
			register size_t len, register uint_t prot);
static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
			uint_t prot);
static int	segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
static size_t	segspt_shmswapout(struct seg *seg);
static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
			register char *vec);
static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len,
			int attr, uint_t flags);
static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
			int attr, int op, ulong_t *lockmap, size_t pos);
static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
			uint_t *protv);
static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
static int segspt_shmgettype(struct seg *seg, caddr_t addr);
static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
			uint_t behav);
static void segspt_shmdump(struct seg *seg);
static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
			struct page ***, enum lock_type, enum seg_rw);
static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t);
static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
static int segspt_shmcapable(struct seg *, segcapability_t);

struct seg_ops segspt_shmops = {
	segspt_shmdup,
	segspt_shmunmap,
	segspt_shmfree,
	segspt_shmfault,
	segspt_shmfaulta,
	segspt_shmsetprot,
	segspt_shmcheckprot,
	segspt_shmkluster,
	segspt_shmswapout,
	segspt_shmsync,
	segspt_shmincore,
	segspt_shmlockop,
	segspt_shmgetprot,
	segspt_shmgetoffset,
	segspt_shmgettype,
	segspt_shmgetvp,
	segspt_shmadvise,	/* advise */
	segspt_shmdump,
	segspt_shmpagelock,
	segspt_shmsetpgsz,
	segspt_shmgetmemid,
	segspt_shmgetpolicy,
	segspt_shmcapable,
};

static void segspt_purge(struct seg *seg);
static int segspt_reclaim(struct seg *, caddr_t, size_t, struct page **,
		enum seg_rw);
static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
		page_t **ppa);



/*ARGSUSED*/
int
sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
	uint_t prot, uint_t flags, uint_t share_szc)
{
	int 	err;
	struct  as	*newas;
	struct	segspt_crargs sptcargs;

#ifdef DEBUG
	TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */,
			tnf_ulong, size, size );
#endif
	if (segspt_minfree == 0)	/* leave min 5% of availrmem for */
		segspt_minfree = availrmem/20;	/* for the system */

	if (!hat_supported(HAT_SHARED_PT, (void *)0))
		return (EINVAL);

	/*
	 * get a new as for this shared memory segment
	 */
	newas = as_alloc();
	newas->a_proc = NULL;
	sptcargs.amp = amp;
	sptcargs.prot = prot;
	sptcargs.flags = flags;
	sptcargs.szc = share_szc;
	/*
	 * create a shared page table (spt) segment
	 */

	if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
		as_free(newas);
		return (err);
	}
	*sptseg = sptcargs.seg_spt;
	return (0);
}

void
sptdestroy(struct as *as, struct anon_map *amp)
{

#ifdef DEBUG
	TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */);
#endif
	(void) as_unmap(as, SEGSPTADDR, amp->size);
	as_free(as);
}

/*
 * called from seg_free().
 * free (i.e., unlock, unmap, return to free list)
 *  all the pages in the given seg.
 */
void
segspt_free(struct seg	*seg)
{
	struct spt_data *sptd = (struct spt_data *)seg->s_data;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));

	if (sptd != NULL) {
		if (sptd->spt_realsize)
			segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);

	if (sptd->spt_ppa_lckcnt)
		kmem_free(sptd->spt_ppa_lckcnt,
		    sizeof (*sptd->spt_ppa_lckcnt)
		    * btopr(sptd->spt_amp->size));
		kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
		cv_destroy(&sptd->spt_cv);
		mutex_destroy(&sptd->spt_lock);
		kmem_free(sptd, sizeof (*sptd));
	}
}

/*ARGSUSED*/
static int
segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
	uint_t flags)
{
	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));

	return (0);
}

/*ARGSUSED*/
static size_t
segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
{
	caddr_t	eo_seg;
	pgcnt_t	npages;
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct seg	*sptseg;
	struct spt_data *sptd;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
#ifdef lint
	seg = seg;
#endif
	sptseg = shmd->shm_sptseg;
	sptd = sptseg->s_data;

	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		eo_seg = addr + len;
		while (addr < eo_seg) {
			/* page exists, and it's locked. */
			*vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
			    SEG_PAGE_ANON;
			addr += PAGESIZE;
		}
		return (len);
	} else {
		struct  anon_map *amp = shmd->shm_amp;
		struct  anon	*ap;
		page_t		*pp;
		pgcnt_t 	anon_index;
		struct vnode 	*vp;
		u_offset_t 	off;
		ulong_t		i;
		int		ret;
		anon_sync_obj_t	cookie;

		addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
		anon_index = seg_page(seg, addr);
		npages = btopr(len);
		if (anon_index + npages > btopr(shmd->shm_amp->size)) {
			return (EINVAL);
		}
		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
		for (i = 0; i < npages; i++, anon_index++) {
			ret = 0;
			anon_array_enter(amp, anon_index, &cookie);
			ap = anon_get_ptr(amp->ahp, anon_index);
			if (ap != NULL) {
				swap_xlate(ap, &vp, &off);
				anon_array_exit(&cookie);
				pp = page_lookup_nowait(vp, off, SE_SHARED);
				if (pp != NULL) {
					ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
					page_unlock(pp);
				}
			} else {
				anon_array_exit(&cookie);
			}
			if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
				ret |= SEG_PAGE_LOCKED;
			}
			*vec++ = (char)ret;
		}
		ANON_LOCK_EXIT(&amp->a_rwlock);
		return (len);
	}
}

static int
segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
{
	size_t share_size;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));

	/*
	 * seg.s_size may have been rounded up to the largest page size
	 * in shmat().
	 * XXX This should be cleanedup. sptdestroy should take a length
	 * argument which should be the same as sptcreate. Then
	 * this rounding would not be needed (or is done in shm.c)
	 * Only the check for full segment will be needed.
	 *
	 * XXX -- shouldn't raddr == 0 always? These tests don't seem
	 * to be useful at all.
	 */
	share_size = page_get_pagesize(seg->s_szc);
	ssize = P2ROUNDUP(ssize, share_size);

	if (raddr == seg->s_base && ssize == seg->s_size) {
		seg_free(seg);
		return (0);
	} else
		return (EINVAL);
}

int
segspt_create(struct seg *seg, caddr_t argsp)
{
	int		err;
	caddr_t		addr = seg->s_base;
	struct spt_data *sptd;
	struct 	segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
	struct anon_map *amp = sptcargs->amp;
	struct kshmid	*sp = amp->a_sp;
	struct	cred	*cred = CRED();
	ulong_t		i, j, anon_index = 0;
	pgcnt_t		npages = btopr(amp->size);
	struct vnode	*vp;
	page_t		**ppa;
	uint_t		hat_flags;
	size_t		pgsz;
	pgcnt_t		pgcnt;
	caddr_t		a;
	pgcnt_t		pidx;
	size_t		sz;
	proc_t		*procp = curproc;
	rctl_qty_t	lockedbytes = 0;
	kproject_t	*proj;

	/*
	 * We are holding the a_lock on the underlying dummy as,
	 * so we can make calls to the HAT layer.
	 */
	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
	ASSERT(sp != NULL);

#ifdef DEBUG
	TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */,
	    tnf_opaque, addr, addr, tnf_ulong, len, seg->s_size);
#endif
	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
		if (err = anon_swap_adjust(npages))
			return (err);
	}
	err = ENOMEM;

	if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
		goto out1;

	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
		if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
		    KM_NOSLEEP)) == NULL)
			goto out2;
	}

	mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);

	if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
		goto out3;

	seg->s_ops = &segspt_ops;
	sptd->spt_vp = vp;
	sptd->spt_amp = amp;
	sptd->spt_prot = sptcargs->prot;
	sptd->spt_flags = sptcargs->flags;
	seg->s_data = (caddr_t)sptd;
	sptd->spt_ppa = NULL;
	sptd->spt_ppa_lckcnt = NULL;
	seg->s_szc = sptcargs->szc;
	cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL);
	sptd->spt_gen = 0;

	ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
	if (seg->s_szc > amp->a_szc) {
		amp->a_szc = seg->s_szc;
	}
	ANON_LOCK_EXIT(&amp->a_rwlock);

	/*
	 * Set policy to affect initial allocation of pages in
	 * anon_map_createpages()
	 */
	(void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
	    NULL, 0, ptob(npages));

	if (sptcargs->flags & SHM_PAGEABLE) {
		size_t  share_sz;
		pgcnt_t new_npgs, more_pgs;
		struct anon_hdr *nahp;
		zone_t *zone;

		share_sz = page_get_pagesize(seg->s_szc);
		if (!IS_P2ALIGNED(amp->size, share_sz)) {
			/*
			 * We are rounding up the size of the anon array
			 * on 4 M boundary because we always create 4 M
			 * of page(s) when locking, faulting pages and we
			 * don't have to check for all corner cases e.g.
			 * if there is enough space to allocate 4 M
			 * page.
			 */
			new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
			more_pgs = new_npgs - npages;

			/*
			 * The zone will never be NULL, as a fully created
			 * shm always has an owning zone.
			 */
			zone = sp->shm_perm.ipc_zone;
			ASSERT(zone != NULL);
			if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
				err = ENOMEM;
				goto out4;
			}

			nahp = anon_create(new_npgs, ANON_SLEEP);
			ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
			(void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
			    ANON_SLEEP);
			anon_release(amp->ahp, npages);
			amp->ahp = nahp;
			ASSERT(amp->swresv == ptob(npages));
			amp->swresv = amp->size = ptob(new_npgs);
			ANON_LOCK_EXIT(&amp->a_rwlock);
			npages = new_npgs;
		}

		sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
		    sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
		sptd->spt_pcachecnt = 0;
		sptd->spt_realsize = ptob(npages);
		sptcargs->seg_spt = seg;
		return (0);
	}

	/*
	 * get array of pages for each anon slot in amp
	 */
	if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
	    seg, addr, S_CREATE, cred)) != 0)
		goto out4;

	mutex_enter(&sp->shm_mlock);

	/* May be partially locked, so, count bytes to charge for locking */
	for (i = 0; i < npages; i++)
		if (ppa[i]->p_lckcnt == 0)
			lockedbytes += PAGESIZE;

	proj = sp->shm_perm.ipc_proj;

	if (lockedbytes > 0) {
		mutex_enter(&procp->p_lock);
		if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
			mutex_exit(&procp->p_lock);
			mutex_exit(&sp->shm_mlock);
			for (i = 0; i < npages; i++)
				page_unlock(ppa[i]);
			err = ENOMEM;
			goto out4;
		}
		mutex_exit(&procp->p_lock);
	}

	/*
	 * addr is initial address corresponding to the first page on ppa list
	 */
	for (i = 0; i < npages; i++) {
		/* attempt to lock all pages */
		if (page_pp_lock(ppa[i], 0, 1) == 0) {
			/*
			 * if unable to lock any page, unlock all
			 * of them and return error
			 */
			for (j = 0; j < i; j++)
				page_pp_unlock(ppa[j], 0, 1);
			for (i = 0; i < npages; i++)
				page_unlock(ppa[i]);
			rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
			mutex_exit(&sp->shm_mlock);
			err = ENOMEM;
			goto out4;
		}
	}
	mutex_exit(&sp->shm_mlock);

	/*
	 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
	 * for the entire life of the segment. For example platforms
	 * that do not support Dynamic Reconfiguration.
	 */
	hat_flags = HAT_LOAD_SHARE;
	if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
		hat_flags |= HAT_LOAD_LOCK;

	/*
	 * Load translations one lare page at a time
	 * to make sure we don't create mappings bigger than
	 * segment's size code in case underlying pages
	 * are shared with segvn's segment that uses bigger
	 * size code than we do.
	 */
	pgsz = page_get_pagesize(seg->s_szc);
	pgcnt = page_get_pagecnt(seg->s_szc);
	for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
		sz = MIN(pgsz, ptob(npages - pidx));
		hat_memload_array(seg->s_as->a_hat, a, sz,
		    &ppa[pidx], sptd->spt_prot, hat_flags);
	}

	/*
	 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
	 * we will leave the pages locked SE_SHARED for the life
	 * of the ISM segment. This will prevent any calls to
	 * hat_pageunload() on this ISM segment for those platforms.
	 */
	if (!(hat_flags & HAT_LOAD_LOCK)) {
		/*
		 * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
		 * we no longer need to hold the SE_SHARED lock on the pages,
		 * since L_PAGELOCK and F_SOFTLOCK calls will grab the
		 * SE_SHARED lock on the pages as necessary.
		 */
		for (i = 0; i < npages; i++)
			page_unlock(ppa[i]);
	}
	sptd->spt_pcachecnt = 0;
	kmem_free(ppa, ((sizeof (page_t *)) * npages));
	sptd->spt_realsize = ptob(npages);
	atomic_add_long(&spt_used, npages);
	sptcargs->seg_spt = seg;
	return (0);

out4:
	seg->s_data = NULL;
	kmem_free(vp, sizeof (*vp));
	cv_destroy(&sptd->spt_cv);
out3:
	mutex_destroy(&sptd->spt_lock);
	if ((sptcargs->flags & SHM_PAGEABLE) == 0)
		kmem_free(ppa, (sizeof (*ppa) * npages));
out2:
	kmem_free(sptd, sizeof (*sptd));
out1:
	if ((sptcargs->flags & SHM_PAGEABLE) == 0)
		anon_swap_restore(npages);
	return (err);
}

/*ARGSUSED*/
void
segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
{
	struct page 	*pp;
	struct spt_data *sptd = (struct spt_data *)seg->s_data;
	pgcnt_t		npages;
	ulong_t		anon_idx;
	struct anon_map *amp;
	struct anon 	*ap;
	struct vnode 	*vp;
	u_offset_t 	off;
	uint_t		hat_flags;
	int		root = 0;
	pgcnt_t		pgs, curnpgs = 0;
	page_t		*rootpp;
	rctl_qty_t	unlocked_bytes = 0;
	kproject_t	*proj;
	kshmid_t	*sp;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));

	len = P2ROUNDUP(len, PAGESIZE);

	npages = btop(len);

	hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP;
	if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) ||
	    (sptd->spt_flags & SHM_PAGEABLE)) {
		hat_flags = HAT_UNLOAD_UNMAP;
	}

	hat_unload(seg->s_as->a_hat, addr, len, hat_flags);

	amp = sptd->spt_amp;
	if (sptd->spt_flags & SHM_PAGEABLE)
		npages = btop(amp->size);

	ASSERT(amp != NULL);

	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		sp = amp->a_sp;
		proj = sp->shm_perm.ipc_proj;
		mutex_enter(&sp->shm_mlock);
	}
	for (anon_idx = 0; anon_idx < npages; anon_idx++) {
		if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
			if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
				panic("segspt_free_pages: null app");
				/*NOTREACHED*/
			}
		} else {
			if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
			    == NULL)
				continue;
		}
		ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
		swap_xlate(ap, &vp, &off);

		/*
		 * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
		 * the pages won't be having SE_SHARED lock at this
		 * point.
		 *
		 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
		 * the pages are still held SE_SHARED locked from the
		 * original segspt_create()
		 *
		 * Our goal is to get SE_EXCL lock on each page, remove
		 * permanent lock on it and invalidate the page.
		 */
		if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
			if (hat_flags == HAT_UNLOAD_UNMAP)
				pp = page_lookup(vp, off, SE_EXCL);
			else {
				if ((pp = page_find(vp, off)) == NULL) {
					panic("segspt_free_pages: "
					    "page not locked");
					/*NOTREACHED*/
				}
				if (!page_tryupgrade(pp)) {
					page_unlock(pp);
					pp = page_lookup(vp, off, SE_EXCL);
				}
			}
			if (pp == NULL) {
				panic("segspt_free_pages: "
				    "page not in the system");
				/*NOTREACHED*/
			}
			ASSERT(pp->p_lckcnt > 0);
			page_pp_unlock(pp, 0, 1);
			if (pp->p_lckcnt == 0)
				unlocked_bytes += PAGESIZE;
		} else {
			if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL)
				continue;
		}
		/*
		 * It's logical to invalidate the pages here as in most cases
		 * these were created by segspt.
		 */
		if (pp->p_szc != 0) {
			if (root == 0) {
				ASSERT(curnpgs == 0);
				root = 1;
				rootpp = pp;
				pgs = curnpgs = page_get_pagecnt(pp->p_szc);
				ASSERT(pgs > 1);
				ASSERT(IS_P2ALIGNED(pgs, pgs));
				ASSERT(!(page_pptonum(pp) & (pgs - 1)));
				curnpgs--;
			} else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
				ASSERT(curnpgs == 1);
				ASSERT(page_pptonum(pp) ==
				    page_pptonum(rootpp) + (pgs - 1));
				page_destroy_pages(rootpp);
				root = 0;
				curnpgs = 0;
			} else {
				ASSERT(curnpgs > 1);
				ASSERT(page_pptonum(pp) ==
				    page_pptonum(rootpp) + (pgs - curnpgs));
				curnpgs--;
			}
		} else {
			if (root != 0 || curnpgs != 0) {
				panic("segspt_free_pages: bad large page");
				/*NOTREACHED*/
			}
			/*LINTED: constant in conditional context */
			VN_DISPOSE(pp, B_INVAL, 0, kcred);
		}
	}
	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		if (unlocked_bytes > 0)
			rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
		mutex_exit(&sp->shm_mlock);
	}
	if (root != 0 || curnpgs != 0) {
		panic("segspt_free_pages: bad large page");
		/*NOTREACHED*/
	}

	/*
	 * mark that pages have been released
	 */
	sptd->spt_realsize = 0;

	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		atomic_add_long(&spt_used, -npages);
		anon_swap_restore(npages);
	}
}

/*
 * Get memory allocation policy info for specified address in given segment
 */
static lgrp_mem_policy_info_t *
segspt_getpolicy(struct seg *seg, caddr_t addr)
{
	struct anon_map		*amp;
	ulong_t			anon_index;
	lgrp_mem_policy_info_t	*policy_info;
	struct spt_data		*spt_data;

	ASSERT(seg != NULL);

	/*
	 * Get anon_map from segspt
	 *
	 * Assume that no lock needs to be held on anon_map, since
	 * it should be protected by its reference count which must be
	 * nonzero for an existing segment
	 * Need to grab readers lock on policy tree though
	 */
	spt_data = (struct spt_data *)seg->s_data;
	if (spt_data == NULL)
		return (NULL);
	amp = spt_data->spt_amp;
	ASSERT(amp->refcnt != 0);

	/*
	 * Get policy info
	 *
	 * Assume starting anon index of 0
	 */
	anon_index = seg_page(seg, addr);
	policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);

	return (policy_info);
}

/*
 * DISM only.
 * Return locked pages over a given range.
 *
 * We will cache all DISM locked pages and save the pplist for the
 * entire segment in the ppa field of the underlying DISM segment structure.
 * Later, during a call to segspt_reclaim() we will use this ppa array
 * to page_unlock() all of the pages and then we will free this ppa list.
 */
/*ARGSUSED*/
static int
segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
    struct page ***ppp, enum lock_type type, enum seg_rw rw)
{
	struct  shm_data *shmd = (struct shm_data *)seg->s_data;
	struct  seg	*sptseg = shmd->shm_sptseg;
	struct  spt_data *sptd = sptseg->s_data;
	pgcnt_t pg_idx, npages, tot_npages, npgs;
	struct  page **pplist, **pl, **ppa, *pp;
	struct  anon_map *amp;
	spgcnt_t	an_idx;
	int 	ret = ENOTSUP;
	uint_t	pl_built = 0;
	struct  anon *ap;
	struct  vnode *vp;
	u_offset_t off;
	pgcnt_t claim_availrmem = 0;
	uint_t	szc;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));

	/*
	 * We want to lock/unlock the entire ISM segment. Therefore,
	 * we will be using the underlying sptseg and it's base address
	 * and length for the caching arguments.
	 */
	ASSERT(sptseg);
	ASSERT(sptd);

	pg_idx = seg_page(seg, addr);
	npages = btopr(len);

	/*
	 * check if the request is larger than number of pages covered
	 * by amp
	 */
	if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
		*ppp = NULL;
		return (ENOTSUP);
	}

	if (type == L_PAGEUNLOCK) {
		ASSERT(sptd->spt_ppa != NULL);

		seg_pinactive(seg, seg->s_base, sptd->spt_amp->size,
		    sptd->spt_ppa, sptd->spt_prot, segspt_reclaim);

		/*
		 * If someone is blocked while unmapping, we purge
		 * segment page cache and thus reclaim pplist synchronously
		 * without waiting for seg_pasync_thread. This speeds up
		 * unmapping in cases where munmap(2) is called, while
		 * raw async i/o is still in progress or where a thread
		 * exits on data fault in a multithreaded application.
		 */
		if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
			segspt_purge(seg);
		}
		return (0);
	} else if (type == L_PAGERECLAIM) {
		ASSERT(sptd->spt_ppa != NULL);
		(void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size,
		    sptd->spt_ppa, sptd->spt_prot);
		return (0);
	}

	if (sptd->spt_flags & DISM_PPA_CHANGED) {
		segspt_purge(seg);
		/*
		 * for DISM ppa needs to be rebuild since
		 * number of locked pages could be changed
		 */
		*ppp = NULL;
		return (ENOTSUP);
	}

	/*
	 * First try to find pages in segment page cache, without
	 * holding the segment lock.
	 */
	pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size,
	    sptd->spt_prot);
	if (pplist != NULL) {
		ASSERT(sptd->spt_ppa != NULL);
		ASSERT(sptd->spt_ppa == pplist);
		ppa = sptd->spt_ppa;
		for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
			if (ppa[an_idx] == NULL) {
				seg_pinactive(seg, seg->s_base,
				    sptd->spt_amp->size, ppa,
				    sptd->spt_prot, segspt_reclaim);
				*ppp = NULL;
				return (ENOTSUP);
			}
			if ((szc = ppa[an_idx]->p_szc) != 0) {
				npgs = page_get_pagecnt(szc);
				an_idx = P2ROUNDUP(an_idx + 1, npgs);
			} else {
				an_idx++;
			}
		}
		/*
		 * Since we cache the entire DISM segment, we want to
		 * set ppp to point to the first slot that corresponds
		 * to the requested addr, i.e. pg_idx.
		 */
		*ppp = &(sptd->spt_ppa[pg_idx]);
		return (0);
	}

	/* The L_PAGELOCK case... */
	mutex_enter(&sptd->spt_lock);
	/*
	 * try to find pages in segment page cache with mutex
	 */
	pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size,
	    sptd->spt_prot);
	if (pplist != NULL) {
		ASSERT(sptd->spt_ppa != NULL);
		ASSERT(sptd->spt_ppa == pplist);
		ppa = sptd->spt_ppa;
		for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
			if (ppa[an_idx] == NULL) {
				mutex_exit(&sptd->spt_lock);
				seg_pinactive(seg, seg->s_base,
				    sptd->spt_amp->size, ppa,
				    sptd->spt_prot, segspt_reclaim);
				*ppp = NULL;
				return (ENOTSUP);
			}
			if ((szc = ppa[an_idx]->p_szc) != 0) {
				npgs = page_get_pagecnt(szc);
				an_idx = P2ROUNDUP(an_idx + 1, npgs);
			} else {
				an_idx++;
			}
		}
		/*
		 * Since we cache the entire DISM segment, we want to
		 * set ppp to point to the first slot that corresponds