xref: /onnv/onnv-gate/usr/src/uts/common/inet/tcp/tcp.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/* Copyright (c) 1990 Mentat Inc. */

#pragma ident	"%Z%%M%	%I%	%E% SMI"
const char tcp_version[] = "%Z%%M%	%I%	%E% SMI";


#include <sys/types.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strsubr.h>
#include <sys/stropts.h>
#include <sys/strlog.h>
#include <sys/strsun.h>
#define	_SUN_TPI_VERSION 2
#include <sys/tihdr.h>
#include <sys/timod.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/suntpi.h>
#include <sys/xti_inet.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/sdt.h>
#include <sys/vtrace.h>
#include <sys/kmem.h>
#include <sys/ethernet.h>
#include <sys/cpuvar.h>
#include <sys/dlpi.h>
#include <sys/multidata.h>
#include <sys/multidata_impl.h>
#include <sys/pattr.h>
#include <sys/policy.h>
#include <sys/priv.h>
#include <sys/zone.h>
#include <sys/sunldi.h>

#include <sys/errno.h>
#include <sys/signal.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/isa_defs.h>
#include <sys/md5.h>
#include <sys/random.h>
#include <sys/sodirect.h>
#include <sys/uio.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <netinet/ip6.h>
#include <netinet/icmp6.h>
#include <net/if.h>
#include <net/route.h>
#include <inet/ipsec_impl.h>

#include <inet/common.h>
#include <inet/ip.h>
#include <inet/ip_impl.h>
#include <inet/ip6.h>
#include <inet/ip_ndp.h>
#include <inet/mi.h>
#include <inet/mib2.h>
#include <inet/nd.h>
#include <inet/optcom.h>
#include <inet/snmpcom.h>
#include <inet/kstatcom.h>
#include <inet/tcp.h>
#include <inet/tcp_impl.h>
#include <net/pfkeyv2.h>
#include <inet/ipsec_info.h>
#include <inet/ipdrop.h>
#include <inet/tcp_trace.h>

#include <inet/ipclassifier.h>
#include <inet/ip_ire.h>
#include <inet/ip_ftable.h>
#include <inet/ip_if.h>
#include <inet/ipp_common.h>
#include <inet/ip_netinfo.h>
#include <sys/squeue.h>
#include <inet/kssl/ksslapi.h>
#include <sys/tsol/label.h>
#include <sys/tsol/tnet.h>
#include <rpc/pmap_prot.h>

/*
 * TCP Notes: aka FireEngine Phase I (PSARC 2002/433)
 *
 * (Read the detailed design doc in PSARC case directory)
 *
 * The entire tcp state is contained in tcp_t and conn_t structure
 * which are allocated in tandem using ipcl_conn_create() and passing
 * IPCL_CONNTCP as a flag. We use 'conn_ref' and 'conn_lock' to protect
 * the references on the tcp_t. The tcp_t structure is never compressed
 * and packets always land on the correct TCP perimeter from the time
 * eager is created till the time tcp_t dies (as such the old mentat
 * TCP global queue is not used for detached state and no IPSEC checking
 * is required). The global queue is still allocated to send out resets
 * for connection which have no listeners and IP directly calls
 * tcp_xmit_listeners_reset() which does any policy check.
 *
 * Protection and Synchronisation mechanism:
 *
 * The tcp data structure does not use any kind of lock for protecting
 * its state but instead uses 'squeues' for mutual exclusion from various
 * read and write side threads. To access a tcp member, the thread should
 * always be behind squeue (via squeue_enter, squeue_enter_nodrain, or
 * squeue_fill). Since the squeues allow a direct function call, caller
 * can pass any tcp function having prototype of edesc_t as argument
 * (different from traditional STREAMs model where packets come in only
 * designated entry points). The list of functions that can be directly
 * called via squeue are listed before the usual function prototype.
 *
 * Referencing:
 *
 * TCP is MT-Hot and we use a reference based scheme to make sure that the
 * tcp structure doesn't disappear when its needed. When the application
 * creates an outgoing connection or accepts an incoming connection, we
 * start out with 2 references on 'conn_ref'. One for TCP and one for IP.
 * The IP reference is just a symbolic reference since ip_tcpclose()
 * looks at tcp structure after tcp_close_output() returns which could
 * have dropped the last TCP reference. So as long as the connection is
 * in attached state i.e. !TCP_IS_DETACHED, we have 2 references on the
 * conn_t. The classifier puts its own reference when the connection is
 * inserted in listen or connected hash. Anytime a thread needs to enter
 * the tcp connection perimeter, it retrieves the conn/tcp from q->ptr
 * on write side or by doing a classify on read side and then puts a
 * reference on the conn before doing squeue_enter/tryenter/fill. For
 * read side, the classifier itself puts the reference under fanout lock
 * to make sure that tcp can't disappear before it gets processed. The
 * squeue will drop this reference automatically so the called function
 * doesn't have to do a DEC_REF.
 *
 * Opening a new connection:
 *
 * The outgoing connection open is pretty simple. tcp_open() does the
 * work in creating the conn/tcp structure and initializing it. The
 * squeue assignment is done based on the CPU the application
 * is running on. So for outbound connections, processing is always done
 * on application CPU which might be different from the incoming CPU
 * being interrupted by the NIC. An optimal way would be to figure out
 * the NIC <-> CPU binding at listen time, and assign the outgoing
 * connection to the squeue attached to the CPU that will be interrupted
 * for incoming packets (we know the NIC based on the bind IP address).
 * This might seem like a problem if more data is going out but the
 * fact is that in most cases the transmit is ACK driven transmit where
 * the outgoing data normally sits on TCP's xmit queue waiting to be
 * transmitted.
 *
 * Accepting a connection:
 *
 * This is a more interesting case because of various races involved in
 * establishing a eager in its own perimeter. Read the meta comment on
 * top of tcp_conn_request(). But briefly, the squeue is picked by
 * ip_tcp_input()/ip_fanout_tcp_v6() based on the interrupted CPU.
 *
 * Closing a connection:
 *
 * The close is fairly straight forward. tcp_close() calls tcp_close_output()
 * via squeue to do the close and mark the tcp as detached if the connection
 * was in state TCPS_ESTABLISHED or greater. In the later case, TCP keep its
 * reference but tcp_close() drop IP's reference always. So if tcp was
 * not killed, it is sitting in time_wait list with 2 reference - 1 for TCP
 * and 1 because it is in classifier's connected hash. This is the condition
 * we use to determine that its OK to clean up the tcp outside of squeue
 * when time wait expires (check the ref under fanout and conn_lock and
 * if it is 2, remove it from fanout hash and kill it).
 *
 * Although close just drops the necessary references and marks the
 * tcp_detached state, tcp_close needs to know the tcp_detached has been
 * set (under squeue) before letting the STREAM go away (because a
 * inbound packet might attempt to go up the STREAM while the close
 * has happened and tcp_detached is not set). So a special lock and
 * flag is used along with a condition variable (tcp_closelock, tcp_closed,
 * and tcp_closecv) to signal tcp_close that tcp_close_out() has marked
 * tcp_detached.
 *
 * Special provisions and fast paths:
 *
 * We make special provision for (AF_INET, SOCK_STREAM) sockets which
 * can't have 'ipv6_recvpktinfo' set and for these type of sockets, IP
 * will never send a M_CTL to TCP. As such, ip_tcp_input() which handles
 * all TCP packets from the wire makes a IPCL_IS_TCP4_CONNECTED_NO_POLICY
 * check to send packets directly to tcp_rput_data via squeue. Everyone
 * else comes through tcp_input() on the read side.
 *
 * We also make special provisions for sockfs by marking tcp_issocket
 * whenever we have only sockfs on top of TCP. This allows us to skip
 * putting the tcp in acceptor hash since a sockfs listener can never
 * become acceptor and also avoid allocating a tcp_t for acceptor STREAM
 * since eager has already been allocated and the accept now happens
 * on acceptor STREAM. There is a big blob of comment on top of
 * tcp_conn_request explaining the new accept. When socket is POP'd,
 * sockfs sends us an ioctl to mark the fact and we go back to old
 * behaviour. Once tcp_issocket is unset, its never set for the
 * life of that connection.
 *
 * In support of on-board asynchronous DMA hardware (e.g. Intel I/OAT)
 * two consoldiation private KAPIs are used to enqueue M_DATA mblk_t's
 * directly to the socket (sodirect) and start an asynchronous copyout
 * to a user-land receive-side buffer (uioa) when a blocking socket read
 * (e.g. read, recv, ...) is pending.
 *
 * This is accomplished when tcp_issocket is set and tcp_sodirect is not
 * NULL so points to an sodirect_t and if marked enabled then we enqueue
 * all mblk_t's directly to the socket.
 *
 * Further, if the sodirect_t sod_uioa and if marked enabled (due to a
 * blocking socket read, e.g. user-land read, recv, ...) then an asynchronous
 * copyout will be started directly to the user-land uio buffer. Also, as we
 * have a pending read, TCP's push logic can take into account the number of
 * bytes to be received and only awake the blocked read()er when the uioa_t
 * byte count has been satisfied.
 *
 * IPsec notes :
 *
 * Since a packet is always executed on the correct TCP perimeter
 * all IPsec processing is defered to IP including checking new
 * connections and setting IPSEC policies for new connection. The
 * only exception is tcp_xmit_listeners_reset() which is called
 * directly from IP and needs to policy check to see if TH_RST
 * can be sent out.
 *
 * PFHooks notes :
 *
 * For mdt case, one meta buffer contains multiple packets. Mblks for every
 * packet are assembled and passed to the hooks. When packets are blocked,
 * or boundary of any packet is changed, the mdt processing is stopped, and
 * packets of the meta buffer are send to the IP path one by one.
 */

/*
 * Values for squeue switch:
 * 1: squeue_enter_nodrain
 * 2: squeue_enter
 * 3: squeue_fill
 */
int tcp_squeue_close = 2;	/* Setable in /etc/system */
int tcp_squeue_wput = 2;

squeue_func_t tcp_squeue_close_proc;
squeue_func_t tcp_squeue_wput_proc;

/*
 * Macros for sodirect:
 *
 * SOD_PTR_ENTER(tcp, sodp) - for the tcp_t pointer "tcp" set the
 * sodirect_t pointer "sodp" to the socket/tcp shared sodirect_t
 * if it exists and is enabled, else to NULL. Note, in the current
 * sodirect implementation the sod_lock must not be held across any
 * STREAMS call (e.g. putnext) else a "recursive mutex_enter" PANIC
 * will result as sod_lock is the streamhead stdata.sd_lock.
 *
 * SOD_NOT_ENABLED(tcp) - return true if not a sodirect tcp_t or the
 * sodirect_t isn't enabled, usefull for ASSERT()ing that a recieve
 * side tcp code path dealing with a tcp_rcv_list or putnext() isn't
 * being used when sodirect code paths should be.
 */

#define	SOD_PTR_ENTER(tcp, sodp)					\
	(sodp) = (tcp)->tcp_sodirect;					\
									\
	if ((sodp) != NULL) {						\
		mutex_enter((sodp)->sod_lock);				\
		if (!((sodp)->sod_state & SOD_ENABLED)) {		\
			mutex_exit((sodp)->sod_lock);			\
			(sodp) = NULL;					\
		}							\
	}

#define	SOD_NOT_ENABLED(tcp)						\
	((tcp)->tcp_sodirect == NULL ||					\
	    !((tcp)->tcp_sodirect->sod_state & SOD_ENABLED))

/*
 * This controls how tiny a write must be before we try to copy it
 * into the the mblk on the tail of the transmit queue.  Not much
 * speedup is observed for values larger than sixteen.  Zero will
 * disable the optimisation.
 */
int tcp_tx_pull_len = 16;

/*
 * TCP Statistics.
 *
 * How TCP statistics work.
 *
 * There are two types of statistics invoked by two macros.
 *
 * TCP_STAT(name) does non-atomic increment of a named stat counter. It is
 * supposed to be used in non MT-hot paths of the code.
 *
 * TCP_DBGSTAT(name) does atomic increment of a named stat counter. It is
 * supposed to be used for DEBUG purposes and may be used on a hot path.
 *
 * Both TCP_STAT and TCP_DBGSTAT counters are available using kstat
 * (use "kstat tcp" to get them).
 *
 * There is also additional debugging facility that marks tcp_clean_death()
 * instances and saves them in tcp_t structure. It is triggered by
 * TCP_TAG_CLEAN_DEATH define. Also, there is a global array of counters for
 * tcp_clean_death() calls that counts the number of times each tag was hit. It
 * is triggered by TCP_CLD_COUNTERS define.
 *
 * How to add new counters.
 *
 * 1) Add a field in the tcp_stat structure describing your counter.
 * 2) Add a line in the template in tcp_kstat2_init() with the name
 *    of the counter.
 *
 *    IMPORTANT!! - make sure that both are in sync !!
 * 3) Use either TCP_STAT or TCP_DBGSTAT with the name.
 *
 * Please avoid using private counters which are not kstat-exported.
 *
 * TCP_TAG_CLEAN_DEATH set to 1 enables tagging of tcp_clean_death() instances
 * in tcp_t structure.
 *
 * TCP_MAX_CLEAN_DEATH_TAG is the maximum number of possible clean death tags.
 */

#ifndef TCP_DEBUG_COUNTER
#ifdef DEBUG
#define	TCP_DEBUG_COUNTER 1
#else
#define	TCP_DEBUG_COUNTER 0
#endif
#endif

#define	TCP_CLD_COUNTERS 0

#define	TCP_TAG_CLEAN_DEATH 1
#define	TCP_MAX_CLEAN_DEATH_TAG 32

#ifdef lint
static int _lint_dummy_;
#endif

#if TCP_CLD_COUNTERS
static uint_t tcp_clean_death_stat[TCP_MAX_CLEAN_DEATH_TAG];
#define	TCP_CLD_STAT(x) tcp_clean_death_stat[x]++
#elif defined(lint)
#define	TCP_CLD_STAT(x) ASSERT(_lint_dummy_ == 0);
#else
#define	TCP_CLD_STAT(x)
#endif

#if TCP_DEBUG_COUNTER
#define	TCP_DBGSTAT(tcps, x)	\
	atomic_add_64(&((tcps)->tcps_statistics.x.value.ui64), 1)
#define	TCP_G_DBGSTAT(x)	\
	atomic_add_64(&(tcp_g_statistics.x.value.ui64), 1)
#elif defined(lint)
#define	TCP_DBGSTAT(tcps, x) ASSERT(_lint_dummy_ == 0);
#define	TCP_G_DBGSTAT(x) ASSERT(_lint_dummy_ == 0);
#else
#define	TCP_DBGSTAT(tcps, x)
#define	TCP_G_DBGSTAT(x)
#endif

#define	TCP_G_STAT(x)	(tcp_g_statistics.x.value.ui64++)

tcp_g_stat_t	tcp_g_statistics;
kstat_t		*tcp_g_kstat;

/*
 * Call either ip_output or ip_output_v6. This replaces putnext() calls on the
 * tcp write side.
 */
#define	CALL_IP_WPUT(connp, q, mp) {					\
	tcp_stack_t	*tcps;						\
									\
	tcps = connp->conn_netstack->netstack_tcp;			\
	ASSERT(((q)->q_flag & QREADR) == 0);				\
	TCP_DBGSTAT(tcps, tcp_ip_output);				\
	connp->conn_send(connp, (mp), (q), IP_WPUT);			\
}

/* Macros for timestamp comparisons */
#define	TSTMP_GEQ(a, b)	((int32_t)((a)-(b)) >= 0)
#define	TSTMP_LT(a, b)	((int32_t)((a)-(b)) < 0)

/*
 * Parameters for TCP Initial Send Sequence number (ISS) generation.  When
 * tcp_strong_iss is set to 1, which is the default, the ISS is calculated
 * by adding three components: a time component which grows by 1 every 4096
 * nanoseconds (versus every 4 microseconds suggested by RFC 793, page 27);
 * a per-connection component which grows by 125000 for every new connection;
 * and an "extra" component that grows by a random amount centered
 * approximately on 64000.  This causes the the ISS generator to cycle every
 * 4.89 hours if no TCP connections are made, and faster if connections are
 * made.
 *
 * When tcp_strong_iss is set to 0, ISS is calculated by adding two
 * components: a time component which grows by 250000 every second; and
 * a per-connection component which grows by 125000 for every new connections.
 *
 * A third method, when tcp_strong_iss is set to 2, for generating ISS is
 * prescribed by Steve Bellovin.  This involves adding time, the 125000 per
 * connection, and a one-way hash (MD5) of the connection ID <sport, dport,
 * src, dst>, a "truly" random (per RFC 1750) number, and a console-entered
 * password.
 */
#define	ISS_INCR	250000
#define	ISS_NSEC_SHT	12

static sin_t	sin_null;	/* Zero address for quick clears */
static sin6_t	sin6_null;	/* Zero address for quick clears */

/*
 * This implementation follows the 4.3BSD interpretation of the urgent
 * pointer and not RFC 1122. Switching to RFC 1122 behavior would cause
 * incompatible changes in protocols like telnet and rlogin.
 */
#define	TCP_OLD_URP_INTERPRETATION	1

#define	TCP_IS_DETACHED_NONEAGER(tcp)	\
	(TCP_IS_DETACHED(tcp) && \
	    (!(tcp)->tcp_hard_binding))

/*
 * TCP reassembly macros.  We hide starting and ending sequence numbers in
 * b_next and b_prev of messages on the reassembly queue.  The messages are
 * chained using b_cont.  These macros are used in tcp_reass() so we don't
 * have to see the ugly casts and assignments.
 */
#define	TCP_REASS_SEQ(mp)		((uint32_t)(uintptr_t)((mp)->b_next))
#define	TCP_REASS_SET_SEQ(mp, u)	((mp)->b_next = \
					(mblk_t *)(uintptr_t)(u))
#define	TCP_REASS_END(mp)		((uint32_t)(uintptr_t)((mp)->b_prev))
#define	TCP_REASS_SET_END(mp, u)	((mp)->b_prev = \
					(mblk_t *)(uintptr_t)(u))

/*
 * Implementation of TCP Timers.
 * =============================
 *
 * INTERFACE:
 *
 * There are two basic functions dealing with tcp timers:
 *
 *	timeout_id_t	tcp_timeout(connp, func, time)
 * 	clock_t		tcp_timeout_cancel(connp, timeout_id)
 *	TCP_TIMER_RESTART(tcp, intvl)
 *
 * tcp_timeout() starts a timer for the 'tcp' instance arranging to call 'func'
 * after 'time' ticks passed. The function called by timeout() must adhere to
 * the same restrictions as a driver soft interrupt handler - it must not sleep
 * or call other functions that might sleep. The value returned is the opaque
 * non-zero timeout identifier that can be passed to tcp_timeout_cancel() to
 * cancel the request. The call to tcp_timeout() may fail in which case it
 * returns zero. This is different from the timeout(9F) function which never
 * fails.
 *
 * The call-back function 'func' always receives 'connp' as its single
 * argument. It is always executed in the squeue corresponding to the tcp
 * structure. The tcp structure is guaranteed to be present at the time the
 * call-back is called.
 *
 * NOTE: The call-back function 'func' is never called if tcp is in
 * 	the TCPS_CLOSED state.
 *
 * tcp_timeout_cancel() attempts to cancel a pending tcp_timeout()
 * request. locks acquired by the call-back routine should not be held across
 * the call to tcp_timeout_cancel() or a deadlock may result.
 *
 * tcp_timeout_cancel() returns -1 if it can not cancel the timeout request.
 * Otherwise, it returns an integer value greater than or equal to 0. In
 * particular, if the call-back function is already placed on the squeue, it can
 * not be canceled.
 *
 * NOTE: both tcp_timeout() and tcp_timeout_cancel() should always be called
 * 	within squeue context corresponding to the tcp instance. Since the
 *	call-back is also called via the same squeue, there are no race
 *	conditions described in untimeout(9F) manual page since all calls are
 *	strictly serialized.
 *
 *      TCP_TIMER_RESTART() is a macro that attempts to cancel a pending timeout
 *	stored in tcp_timer_tid and starts a new one using
 *	MSEC_TO_TICK(intvl). It always uses tcp_timer() function as a call-back
 *	and stores the return value of tcp_timeout() in the tcp->tcp_timer_tid
 *	field.
 *
 * NOTE: since the timeout cancellation is not guaranteed, the cancelled
 *	call-back may still be called, so it is possible tcp_timer() will be
 *	called several times. This should not be a problem since tcp_timer()
 *	should always check the tcp instance state.
 *
 *
 * IMPLEMENTATION:
 *
 * TCP timers are implemented using three-stage process. The call to
 * tcp_timeout() uses timeout(9F) function to call tcp_timer_callback() function
 * when the timer expires. The tcp_timer_callback() arranges the call of the
 * tcp_timer_handler() function via squeue corresponding to the tcp
 * instance. The tcp_timer_handler() calls actual requested timeout call-back
 * and passes tcp instance as an argument to it. Information is passed between
 * stages using the tcp_timer_t structure which contains the connp pointer, the
 * tcp call-back to call and the timeout id returned by the timeout(9F).
 *
 * The tcp_timer_t structure is not used directly, it is embedded in an mblk_t -
 * like structure that is used to enter an squeue. The mp->b_rptr of this pseudo
 * mblk points to the beginning of tcp_timer_t structure. The tcp_timeout()
 * returns the pointer to this mblk.
 *
 * The pseudo mblk is allocated from a special tcp_timer_cache kmem cache. It
 * looks like a normal mblk without actual dblk attached to it.
 *
 * To optimize performance each tcp instance holds a small cache of timer
 * mblocks. In the current implementation it caches up to two timer mblocks per
 * tcp instance. The cache is preserved over tcp frees and is only freed when
 * the whole tcp structure is destroyed by its kmem destructor. Since all tcp
 * timer processing happens on a corresponding squeue, the cache manipulation
 * does not require any locks. Experiments show that majority of timer mblocks
 * allocations are satisfied from the tcp cache and do not involve kmem calls.
 *
 * The tcp_timeout() places a refhold on the connp instance which guarantees
 * that it will be present at the time the call-back function fires. The
 * tcp_timer_handler() drops the reference after calling the call-back, so the
 * call-back function does not need to manipulate the references explicitly.
 */

typedef struct tcp_timer_s {
	conn_t	*connp;
	void 	(*tcpt_proc)(void *);
	timeout_id_t   tcpt_tid;
} tcp_timer_t;

static kmem_cache_t *tcp_timercache;
kmem_cache_t	*tcp_sack_info_cache;
kmem_cache_t	*tcp_iphc_cache;

/*
 * For scalability, we must not run a timer for every TCP connection
 * in TIME_WAIT state.  To see why, consider (for time wait interval of
 * 4 minutes):
 *	1000 connections/sec * 240 seconds/time wait = 240,000 active conn's
 *
 * This list is ordered by time, so you need only delete from the head
 * until you get to entries which aren't old enough to delete yet.
 * The list consists of only the detached TIME_WAIT connections.
 *
 * Note that the timer (tcp_time_wait_expire) is started when the tcp_t
 * becomes detached TIME_WAIT (either by changing the state and already
 * being detached or the other way around). This means that the TIME_WAIT
 * state can be extended (up to doubled) if the connection doesn't become
 * detached for a long time.
 *
 * The list manipulations (including tcp_time_wait_next/prev)
 * are protected by the tcp_time_wait_lock. The content of the
 * detached TIME_WAIT connections is protected by the normal perimeters.
 *
 * This list is per squeue and squeues are shared across the tcp_stack_t's.
 * Things on tcp_time_wait_head remain associated with the tcp_stack_t
 * and conn_netstack.
 * The tcp_t's that are added to tcp_free_list are disassociated and
 * have NULL tcp_tcps and conn_netstack pointers.
 */
typedef struct tcp_squeue_priv_s {
	kmutex_t	tcp_time_wait_lock;
	timeout_id_t	tcp_time_wait_tid;
	tcp_t		*tcp_time_wait_head;
	tcp_t		*tcp_time_wait_tail;
	tcp_t		*tcp_free_list;
	uint_t		tcp_free_list_cnt;
} tcp_squeue_priv_t;

/*
 * TCP_TIME_WAIT_DELAY governs how often the time_wait_collector runs.
 * Running it every 5 seconds seems to give the best results.
 */
#define	TCP_TIME_WAIT_DELAY drv_usectohz(5000000)

/*
 * To prevent memory hog, limit the number of entries in tcp_free_list
 * to 1% of available memory / number of cpus
 */
uint_t tcp_free_list_max_cnt = 0;

#define	TCP_XMIT_LOWATER	4096
#define	TCP_XMIT_HIWATER	49152
#define	TCP_RECV_LOWATER	2048
#define	TCP_RECV_HIWATER	49152

/*
 *  PAWS needs a timer for 24 days.  This is the number of ticks in 24 days
 */
#define	PAWS_TIMEOUT	((clock_t)(24*24*60*60*hz))

#define	TIDUSZ	4096	/* transport interface data unit size */

/*
 * Bind hash list size and has function.  It has to be a power of 2 for
 * hashing.
 */
#define	TCP_BIND_FANOUT_SIZE	512
#define	TCP_BIND_HASH(lport) (ntohs(lport) & (TCP_BIND_FANOUT_SIZE - 1))
/*
 * Size of listen and acceptor hash list.  It has to be a power of 2 for
 * hashing.
 */
#define	TCP_FANOUT_SIZE		256

#ifdef	_ILP32
#define	TCP_ACCEPTOR_HASH(accid)					\
		(((uint_t)(accid) >> 8) & (TCP_FANOUT_SIZE - 1))
#else
#define	TCP_ACCEPTOR_HASH(accid)					\
		((uint_t)(accid) & (TCP_FANOUT_SIZE - 1))
#endif	/* _ILP32 */

#define	IP_ADDR_CACHE_SIZE	2048
#define	IP_ADDR_CACHE_HASH(faddr)					\
	(ntohl(faddr) & (IP_ADDR_CACHE_SIZE -1))

/* Hash for HSPs uses all 32 bits, since both networks and hosts are in table */
#define	TCP_HSP_HASH_SIZE 256

#define	TCP_HSP_HASH(addr)					\
	(((addr>>24) ^ (addr >>16) ^			\
	    (addr>>8) ^ (addr)) % TCP_HSP_HASH_SIZE)

/*
 * TCP options struct returned from tcp_parse_options.
 */
typedef struct tcp_opt_s {
	uint32_t	tcp_opt_mss;
	uint32_t	tcp_opt_wscale;
	uint32_t	tcp_opt_ts_val;
	uint32_t	tcp_opt_ts_ecr;
	tcp_t		*tcp;
} tcp_opt_t;

/*
 * RFC1323-recommended phrasing of TSTAMP option, for easier parsing
 */

#ifdef _BIG_ENDIAN
#define	TCPOPT_NOP_NOP_TSTAMP ((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | \
	(TCPOPT_TSTAMP << 8) | 10)
#else
#define	TCPOPT_NOP_NOP_TSTAMP ((10 << 24) | (TCPOPT_TSTAMP << 16) | \
	(TCPOPT_NOP << 8) | TCPOPT_NOP)
#endif

/*
 * Flags returned from tcp_parse_options.
 */
#define	TCP_OPT_MSS_PRESENT	1
#define	TCP_OPT_WSCALE_PRESENT	2
#define	TCP_OPT_TSTAMP_PRESENT	4
#define	TCP_OPT_SACK_OK_PRESENT	8
#define	TCP_OPT_SACK_PRESENT	16

/* TCP option length */
#define	TCPOPT_NOP_LEN		1
#define	TCPOPT_MAXSEG_LEN	4
#define	TCPOPT_WS_LEN		3
#define	TCPOPT_REAL_WS_LEN	(TCPOPT_WS_LEN+1)
#define	TCPOPT_TSTAMP_LEN	10
#define	TCPOPT_REAL_TS_LEN	(TCPOPT_TSTAMP_LEN+2)
#define	TCPOPT_SACK_OK_LEN	2
#define	TCPOPT_REAL_SACK_OK_LEN	(TCPOPT_SACK_OK_LEN+2)
#define	TCPOPT_REAL_SACK_LEN	4
#define	TCPOPT_MAX_SACK_LEN	36
#define	TCPOPT_HEADER_LEN	2

/* TCP cwnd burst factor. */
#define	TCP_CWND_INFINITE	65535
#define	TCP_CWND_SS		3
#define	TCP_CWND_NORMAL		5

/* Maximum TCP initial cwin (start/restart). */
#define	TCP_MAX_INIT_CWND	8

/*
 * Initialize cwnd according to RFC 3390.  def_max_init_cwnd is
 * either tcp_slow_start_initial or tcp_slow_start_after idle
 * depending on the caller.  If the upper layer has not used the
 * TCP_INIT_CWND option to change the initial cwnd, tcp_init_cwnd
 * should be 0 and we use the formula in RFC 3390 to set tcp_cwnd.
 * If the upper layer has changed set the tcp_init_cwnd, just use
 * it to calculate the tcp_cwnd.
 */
#define	SET_TCP_INIT_CWND(tcp, mss, def_max_init_cwnd)			\
{									\
	if ((tcp)->tcp_init_cwnd == 0) {				\
		(tcp)->tcp_cwnd = MIN(def_max_init_cwnd * (mss),	\
		    MIN(4 * (mss), MAX(2 * (mss), 4380 / (mss) * (mss)))); \
	} else {							\
		(tcp)->tcp_cwnd = (tcp)->tcp_init_cwnd * (mss);		\
	}								\
	tcp->tcp_cwnd_cnt = 0;						\
}

/* TCP Timer control structure */
typedef struct tcpt_s {
	pfv_t	tcpt_pfv;	/* The routine we are to call */
	tcp_t	*tcpt_tcp;	/* The parameter we are to pass in */
} tcpt_t;

/* Host Specific Parameter structure */
typedef struct tcp_hsp {
	struct tcp_hsp	*tcp_hsp_next;
	in6_addr_t	tcp_hsp_addr_v6;
	in6_addr_t	tcp_hsp_subnet_v6;
	uint_t		tcp_hsp_vers;	/* IPV4_VERSION | IPV6_VERSION */
	int32_t		tcp_hsp_sendspace;
	int32_t		tcp_hsp_recvspace;
	int32_t		tcp_hsp_tstamp;
} tcp_hsp_t;
#define	tcp_hsp_addr	V4_PART_OF_V6(tcp_hsp_addr_v6)
#define	tcp_hsp_subnet	V4_PART_OF_V6(tcp_hsp_subnet_v6)

/*
 * Functions called directly via squeue having a prototype of edesc_t.
 */
void		tcp_conn_request(void *arg, mblk_t *mp, void *arg2);
static void	tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2);
void		tcp_accept_finish(void *arg, mblk_t *mp, void *arg2);
static void	tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2);
static void	tcp_wput_proto(void *arg, mblk_t *mp, void *arg2);
void 		tcp_input(void *arg, mblk_t *mp, void *arg2);
void		tcp_rput_data(void *arg, mblk_t *mp, void *arg2);
static void	tcp_close_output(void *arg, mblk_t *mp, void *arg2);
void		tcp_output(void *arg, mblk_t *mp, void *arg2);
static void	tcp_rsrv_input(void *arg, mblk_t *mp, void *arg2);
static void	tcp_timer_handler(void *arg, mblk_t *mp, void *arg2);
static void	tcp_linger_interrupted(void *arg, mblk_t *mp, void *arg2);


/* Prototype for TCP functions */
static void	tcp_random_init(void);
int		tcp_random(void);
static void	tcp_accept(tcp_t *tcp, mblk_t *mp);
static void	tcp_accept_swap(tcp_t *listener, tcp_t *acceptor,
		    tcp_t *eager);
static int	tcp_adapt_ire(tcp_t *tcp, mblk_t *ire_mp);
static in_port_t tcp_bindi(tcp_t *tcp, in_port_t port, const in6_addr_t *laddr,
    int reuseaddr, boolean_t quick_connect, boolean_t bind_to_req_port_only,
    boolean_t user_specified);
static void	tcp_closei_local(tcp_t *tcp);
static void	tcp_close_detached(tcp_t *tcp);
static boolean_t tcp_conn_con(tcp_t *tcp, uchar_t *iphdr, tcph_t *tcph,
			mblk_t *idmp, mblk_t **defermp);
static void	tcp_connect(tcp_t *tcp, mblk_t *mp);
static void	tcp_connect_ipv4(tcp_t *tcp, mblk_t *mp, ipaddr_t *dstaddrp,
		    in_port_t dstport, uint_t srcid);
static void	tcp_connect_ipv6(tcp_t *tcp, mblk_t *mp, in6_addr_t *dstaddrp,
		    in_port_t dstport, uint32_t flowinfo, uint_t srcid,
		    uint32_t scope_id);
static int	tcp_clean_death(tcp_t *tcp, int err, uint8_t tag);
static void	tcp_def_q_set(tcp_t *tcp, mblk_t *mp);
static void	tcp_disconnect(tcp_t *tcp, mblk_t *mp);
static char	*tcp_display(tcp_t *tcp, char *, char);
static boolean_t tcp_eager_blowoff(tcp_t *listener, t_scalar_t seqnum);
static void	tcp_eager_cleanup(tcp_t *listener, boolean_t q0_only);
static void	tcp_eager_unlink(tcp_t *tcp);
static void	tcp_err_ack(tcp_t *tcp, mblk_t *mp, int tlierr,
		    int unixerr);
static void	tcp_err_ack_prim(tcp_t *tcp, mblk_t *mp, int primitive,
		    int tlierr, int unixerr);
static int	tcp_extra_priv_ports_get(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static int	tcp_extra_priv_ports_add(queue_t *q, mblk_t *mp,
		    char *value, caddr_t cp, cred_t *cr);
static int	tcp_extra_priv_ports_del(queue_t *q, mblk_t *mp,
		    char *value, caddr_t cp, cred_t *cr);
static int	tcp_tpistate(tcp_t *tcp);
static void	tcp_bind_hash_insert(tf_t *tf, tcp_t *tcp,
    int caller_holds_lock);
static void	tcp_bind_hash_remove(tcp_t *tcp);
static tcp_t	*tcp_acceptor_hash_lookup(t_uscalar_t id, tcp_stack_t *);
void		tcp_acceptor_hash_insert(t_uscalar_t id, tcp_t *tcp);
static void	tcp_acceptor_hash_remove(tcp_t *tcp);
static void	tcp_capability_req(tcp_t *tcp, mblk_t *mp);
static void	tcp_info_req(tcp_t *tcp, mblk_t *mp);
static void	tcp_addr_req(tcp_t *tcp, mblk_t *mp);
static void	tcp_addr_req_ipv6(tcp_t *tcp, mblk_t *mp);
void		tcp_g_q_setup(tcp_stack_t *);
void		tcp_g_q_create(tcp_stack_t *);
void		tcp_g_q_destroy(tcp_stack_t *);
static int	tcp_header_init_ipv4(tcp_t *tcp);
static int	tcp_header_init_ipv6(tcp_t *tcp);
int		tcp_init(tcp_t *tcp, queue_t *q);
static int	tcp_init_values(tcp_t *tcp);
static mblk_t	*tcp_ip_advise_mblk(void *addr, int addr_len, ipic_t **ipic);
static mblk_t	*tcp_ip_bind_mp(tcp_t *tcp, t_scalar_t bind_prim,
		    t_scalar_t addr_length);
static void	tcp_ip_ire_mark_advice(tcp_t *tcp);
static void	tcp_ip_notify(tcp_t *tcp);
static mblk_t	*tcp_ire_mp(mblk_t *mp);
static void	tcp_iss_init(tcp_t *tcp);
static void	tcp_keepalive_killer(void *arg);
static int	tcp_parse_options(tcph_t *tcph, tcp_opt_t *tcpopt);
static void	tcp_mss_set(tcp_t *tcp, uint32_t size, boolean_t do_ss);
static int	tcp_conprim_opt_process(tcp_t *tcp, mblk_t *mp,
		    int *do_disconnectp, int *t_errorp, int *sys_errorp);
static boolean_t tcp_allow_connopt_set(int level, int name);
int		tcp_opt_default(queue_t *q, int level, int name, uchar_t *ptr);
int		tcp_opt_get(queue_t *q, int level, int name, uchar_t *ptr);
int		tcp_opt_set(queue_t *q, uint_t optset_context, int level,
		    int name, uint_t inlen, uchar_t *invalp, uint_t *outlenp,
		    uchar_t *outvalp, void *thisdg_attrs, cred_t *cr,
		    mblk_t *mblk);
static void	tcp_opt_reverse(tcp_t *tcp, ipha_t *ipha);
static int	tcp_opt_set_header(tcp_t *tcp, boolean_t checkonly,
		    uchar_t *ptr, uint_t len);
static int	tcp_param_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr);
static boolean_t tcp_param_register(IDP *ndp, tcpparam_t *tcppa, int cnt,
    tcp_stack_t *);
static int	tcp_param_set(queue_t *q, mblk_t *mp, char *value,
		    caddr_t cp, cred_t *cr);
static int	tcp_param_set_aligned(queue_t *q, mblk_t *mp, char *value,
		    caddr_t cp, cred_t *cr);
static void	tcp_iss_key_init(uint8_t *phrase, int len, tcp_stack_t *);
static int	tcp_1948_phrase_set(queue_t *q, mblk_t *mp, char *value,
		    caddr_t cp, cred_t *cr);
static void	tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_cnt);
static mblk_t	*tcp_reass(tcp_t *tcp, mblk_t *mp, uint32_t start);
static void	tcp_reass_elim_overlap(tcp_t *tcp, mblk_t *mp);
static void	tcp_reinit(tcp_t *tcp);
static void	tcp_reinit_values(tcp_t *tcp);
static void	tcp_report_item(mblk_t *mp, tcp_t *tcp, int hashval,
		    tcp_t *thisstream, cred_t *cr);

static uint_t	tcp_rcv_drain(queue_t *q, tcp_t *tcp);
static void	tcp_sack_rxmit(tcp_t *tcp, uint_t *flags);
static boolean_t tcp_send_rst_chk(tcp_stack_t *);
static void	tcp_ss_rexmit(tcp_t *tcp);
static mblk_t	*tcp_rput_add_ancillary(tcp_t *tcp, mblk_t *mp, ip6_pkt_t *ipp);
static void	tcp_process_options(tcp_t *, tcph_t *);
static void	tcp_rput_common(tcp_t *tcp, mblk_t *mp);
static void	tcp_rsrv(queue_t *q);
static int	tcp_rwnd_set(tcp_t *tcp, uint32_t rwnd);
static int	tcp_snmp_state(tcp_t *tcp);
static int	tcp_status_report(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static int	tcp_bind_hash_report(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static int	tcp_listen_hash_report(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static int	tcp_conn_hash_report(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static int	tcp_acceptor_hash_report(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static int	tcp_host_param_set(queue_t *q, mblk_t *mp, char *value,
		    caddr_t cp, cred_t *cr);
static int	tcp_host_param_set_ipv6(queue_t *q, mblk_t *mp, char *value,
		    caddr_t cp, cred_t *cr);
static int	tcp_host_param_report(queue_t *q, mblk_t *mp, caddr_t cp,
		    cred_t *cr);
static void	tcp_timer(void *arg);
static void	tcp_timer_callback(void *);
static in_port_t tcp_update_next_port(in_port_t port, const tcp_t *tcp,
    boolean_t random);
static in_port_t tcp_get_next_priv_port(const tcp_t *);
static void	tcp_wput_sock(queue_t *q, mblk_t *mp);
void		tcp_wput_accept(queue_t *q, mblk_t *mp);
static void	tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent);
static void	tcp_wput_flush(tcp_t *tcp, mblk_t *mp);
static void	tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp);
static int	tcp_send(queue_t *q, tcp_t *tcp, const int mss,
		    const int tcp_hdr_len, const int tcp_tcp_hdr_len,
		    const int num_sack_blk, int *usable, uint_t *snxt,
		    int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time,
		    const int mdt_thres);
static int	tcp_multisend(queue_t *q, tcp_t *tcp, const int mss,
		    const int tcp_hdr_len, const int tcp_tcp_hdr_len,
		    const int num_sack_blk, int *usable, uint_t *snxt,
		    int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time,
		    const int mdt_thres);
static void	tcp_fill_header(tcp_t *tcp, uchar_t *rptr, clock_t now,
		    int num_sack_blk);
static void	tcp_wsrv(queue_t *q);
static int	tcp_xmit_end(tcp_t *tcp);
static void	tcp_ack_timer(void *arg);
static mblk_t	*tcp_ack_mp(tcp_t *tcp);
static void	tcp_xmit_early_reset(char *str, mblk_t *mp,
		    uint32_t seq, uint32_t ack, int ctl, uint_t ip_hdr_len,
		    zoneid_t zoneid, tcp_stack_t *, conn_t *connp);
static void	tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq,
		    uint32_t ack, int ctl);
static tcp_hsp_t *tcp_hsp_lookup(ipaddr_t addr, tcp_stack_t *);
static tcp_hsp_t *tcp_hsp_lookup_ipv6(in6_addr_t *addr, tcp_stack_t *);
static int	setmaxps(queue_t *q, int maxpsz);
static void	tcp_set_rto(tcp_t *, time_t);
static boolean_t tcp_check_policy(tcp_t *, mblk_t *, ipha_t *, ip6_t *,
		    boolean_t, boolean_t);
static void	tcp_icmp_error_ipv6(tcp_t *tcp, mblk_t *mp,
		    boolean_t ipsec_mctl);
static mblk_t	*tcp_setsockopt_mp(int level, int cmd,
		    char *opt, int optlen);
static int	tcp_build_hdrs(queue_t *, tcp_t *);
static void	tcp_time_wait_processing(tcp_t *tcp, mblk_t *mp,
		    uint32_t seg_seq, uint32_t seg_ack, int seg_len,
		    tcph_t *tcph);
boolean_t	tcp_paws_check(tcp_t *tcp, tcph_t *tcph, tcp_opt_t *tcpoptp);
boolean_t	tcp_reserved_port_add(int, in_port_t *, in_port_t *);
boolean_t	tcp_reserved_port_del(in_port_t, in_port_t);
boolean_t	tcp_reserved_port_check(in_port_t, tcp_stack_t *);
static tcp_t	*tcp_alloc_temp_tcp(in_port_t, tcp_stack_t *);
static int	tcp_reserved_port_list(queue_t *, mblk_t *, caddr_t, cred_t *);
static mblk_t	*tcp_mdt_info_mp(mblk_t *);
static void	tcp_mdt_update(tcp_t *, ill_mdt_capab_t *, boolean_t);
static int	tcp_mdt_add_attrs(multidata_t *, const mblk_t *,
		    const boolean_t, const uint32_t, const uint32_t,
		    const uint32_t, const uint32_t, tcp_stack_t *);
static void	tcp_multisend_data(tcp_t *, ire_t *, const ill_t *, mblk_t *,
		    const uint_t, const uint_t, boolean_t *);
static mblk_t	*tcp_lso_info_mp<