common/os/ftrace.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/types.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/cpuvar.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/debug.h>
#include <sys/param.h>
#include <sys/atomic.h>
#include <sys/ftrace.h>

/*
 * Tunable parameters:
 *
 * ftrace_atboot	- whether to start fast tracing at boot.
 * ftrace_nent		- size of the per-CPU event ring buffer.
 */
int ftrace_atboot = 0;
int ftrace_nent = FTRACE_NENT;

/*
 * Global Tracing State:
 *
 *                NOTREADY(=0)
 *                  |
 *            ftrace_init()
 *                  |
 *                  |
 *                  v
 *      +-------->READY-------+
 *      |                     |
 *  ftrace_stop()         ftrace_start()
 *      |                     |
 *      +---(ENABLED|READY)<--+
 *
 * During boot, ftrace_init() is called and the state becomes
 * READY. If ftrace_atboot is set, ftrace_start() is called at
 * this time.
 *
 * If FTRACE_READY is set, then tracing can be enabled.
 * If FTRACE_ENABLED is set, tracing is enabled on the set of CPUs
 *   which are currently FTRACE_READY.
 */
static int ftrace_state = 0;

/*
 * Per-CPU Tracing State:
 *
 *     +-----------------READY<--------------+
 *     |                 ^   |               |
 *     |                 | ftrace_cpu_fini() |
 *     |                 |   |               |
 *     |   ftrace_cpu_init() |               |
 *     |                 |   v     ftrace_cpu_stop()
 *     |              NOTREADY(=0)           |
 *     |                   ^                 |
 * ftrace_cpu_start()      |                 |
 *     |              ftrace_cpu_fini()      |
 *     |                   |                 |
 *     +----------->(ENABLED|READY)----------+
 *
 */

/*
 * Locking :
 *
 * Trace context code does not use any lock. There is a per-cpu circular trace
 * buffer that has a head, a tail and a current pointer. Each record of this
 * buffer is of equal length. Before doing anything, trace context code checks
 * the per-cpu ENABLED bit. Trace buffer is allocated in non-trace context and
 * it sets this bit only after allocating and setting up the buffer. So trace
 * context code can't access the buffer till it is set up completely. The
 * buffer is freed also in non-trace context. The code that frees the buffer is
 * executed only after the corresponding cpu is powered off. So when this
 * happens, no trace context code can be running on it. We only need to make
 * sure that trace context code is not preempted from the cpu in the middle of
 * accessing the trace buffer. This can be achieved simply by disabling
 * interrupts temporarily. This approach makes the least assumption about the
 * state of the callers of tracing functions.
 *
 * A single global lock, ftrace_lock protects assignments to all global and
 * per-cpu trace variables. It does not protect reading of those in some cases.
 *
 * More specifically, it protects assignments to:
 *
 *   ftrace_state
 *   cpu[N]->cpu_ftrace.ftd_state
 *   cpu[N]->cpu_ftrace.ftd_first
 *   cpu[N]->cpu_ftrace.ftd_last
 *
 * Does _not_ protect reading of cpu[N]->cpu_ftrace.ftd_state
 * Does _not_ protect cpu[N]->cpu_ftrace.ftd_cur
 * Does _not_ protect reading of ftrace_state
 */
static kmutex_t ftrace_lock;

/*
 * Check whether a CPU is installed.
 */
#define	IS_CPU(i) (cpu[i] != NULL)

static void
ftrace_cpu_init(int cpuid)
{
	ftrace_data_t *ftd;

	/*
	 * This can be called with "cpu[cpuid]->cpu_flags & CPU_EXISTS"
	 * being false - e.g. when a CPU is DR'ed in.
	 */
	ASSERT(MUTEX_HELD(&ftrace_lock));
	ASSERT(IS_CPU(cpuid));

	ftd = &cpu[cpuid]->cpu_ftrace;
	if (ftd->ftd_state & FTRACE_READY)
		return;

	/*
	 * We don't allocate the buffers until the first time
	 *   ftrace_cpu_start() is called, so that they're not
	 *   allocated if ftrace is never enabled.
	 */
	ftd->ftd_state |= FTRACE_READY;
	ASSERT(!(ftd->ftd_state & FTRACE_ENABLED));
}

/*
 * Only called from cpu_unconfigure() (and cpu_configure() on error).
 * At this point, cpu[cpuid] is about to be freed and NULLed out,
 *   so we'd better clean up after ourselves.
 */
static void
ftrace_cpu_fini(int cpuid)
{
	ftrace_data_t *ftd;

	ASSERT(MUTEX_HELD(&ftrace_lock));
	ASSERT(IS_CPU(cpuid));
	ASSERT((cpu[cpuid]->cpu_flags & CPU_POWEROFF) != 0);

	ftd = &cpu[cpuid]->cpu_ftrace;
	if (!(ftd->ftd_state & FTRACE_READY))
		return;

	/*
	 * This cpu is powered off and no code can be executing on it. So
	 * we can simply finish our cleanup. There is no need for a xcall
	 * to make sure that this cpu is out of trace context.
	 *
	 * The cpu structure will be cleared soon. But, for the sake of
	 * debugging, clear our pointers and state.
	 */
	if (ftd->ftd_first != NULL) {
		kmem_free(ftd->ftd_first,
		    ftrace_nent * sizeof (ftrace_record_t));
	}
	bzero(ftd, sizeof (ftrace_data_t));
}

static void
ftrace_cpu_start(int cpuid)
{
	ftrace_data_t *ftd;

	ASSERT(MUTEX_HELD(&ftrace_lock));
	ASSERT(IS_CPU(cpuid));
	ASSERT(ftrace_state & FTRACE_ENABLED);

	ftd = &cpu[cpuid]->cpu_ftrace;
	if (ftd->ftd_state & FTRACE_READY) {
		if (ftd->ftd_first == NULL) {
			ftrace_record_t *ptrs;

			mutex_exit(&ftrace_lock);
			ptrs = kmem_zalloc(ftrace_nent *
			    sizeof (ftrace_record_t), KM_SLEEP);
			mutex_enter(&ftrace_lock);
			if (ftd->ftd_first != NULL) {
				/*
				 * Someone else beat us to it. The winner will
				 * set up the pointers and the state.
				 */
				kmem_free(ptrs,
				    ftrace_nent * sizeof (ftrace_record_t));
				return;
			}

			ftd->ftd_first = ptrs;
			ftd->ftd_last = ptrs + (ftrace_nent - 1);
			ftd->ftd_cur = ptrs;
			membar_producer();
		}
		ftd->ftd_state |= FTRACE_ENABLED;
	}
}

static void
ftrace_cpu_stop(int cpuid)
{
	ASSERT(MUTEX_HELD(&ftrace_lock));
	ASSERT(IS_CPU(cpuid));
	cpu[cpuid]->cpu_ftrace.ftd_state &= ~(FTRACE_ENABLED);
}

/*
 * Hook for DR.
 */
/*ARGSUSED*/
int
ftrace_cpu_setup(cpu_setup_t what, int id, void *arg)
{
	if (!(ftrace_state & FTRACE_READY))
		return (0);

	switch (what) {
	case CPU_CONFIG:
		mutex_enter(&ftrace_lock);
		ftrace_cpu_init(id);
		if (ftrace_state & FTRACE_ENABLED)
			ftrace_cpu_start(id);
		mutex_exit(&ftrace_lock);
		break;

	case CPU_UNCONFIG:
		mutex_enter(&ftrace_lock);
		ftrace_cpu_fini(id);
		mutex_exit(&ftrace_lock);
		break;

	default:
		break;
	}
	return (0);
}

void
ftrace_init(void)
{
	int i;

	ASSERT(!(ftrace_state & FTRACE_READY));
	mutex_init(&ftrace_lock, NULL, MUTEX_DEFAULT, NULL);

	mutex_enter(&ftrace_lock);
	for (i = 0; i < NCPU; i++) {
		if (IS_CPU(i)) {
			/* should have been kmem_zalloc()'ed */
			ASSERT(cpu[i]->cpu_ftrace.ftd_state == 0);
			ASSERT(cpu[i]->cpu_ftrace.ftd_first == NULL);
			ASSERT(cpu[i]->cpu_ftrace.ftd_last == NULL);
			ASSERT(cpu[i]->cpu_ftrace.ftd_cur == NULL);
		}
	}

	if (ftrace_nent < 1) {
		mutex_exit(&ftrace_lock);
		return;
	}

	for (i = 0; i < NCPU; i++)
		if (IS_CPU(i))
			ftrace_cpu_init(i);

	ftrace_state |= FTRACE_READY;
	mutex_enter(&cpu_lock);
	register_cpu_setup_func(ftrace_cpu_setup, NULL);
	mutex_exit(&cpu_lock);
	mutex_exit(&ftrace_lock);

	if (ftrace_atboot)
		(void) ftrace_start();
}

/*
 * Called from uadmin ioctl, or via mp_init_table[] during boot.
 */
int
ftrace_start(void)
{
	int i, was_enabled = 0;

	if (ftrace_state & FTRACE_READY) {
		mutex_enter(&ftrace_lock);
		was_enabled = ((ftrace_state & FTRACE_ENABLED) != 0);
		ftrace_state |= FTRACE_ENABLED;
		for (i = 0; i < NCPU; i++)
			if (IS_CPU(i))
				ftrace_cpu_start(i);
		mutex_exit(&ftrace_lock);
	}

	return (was_enabled);
}

/*
 * Called from uadmin ioctl, to stop tracing.
 */
int
ftrace_stop(void)
{
	int i, was_enabled = 0;

	if (ftrace_state & FTRACE_READY) {
		mutex_enter(&ftrace_lock);
		if (ftrace_state & FTRACE_ENABLED) {
			was_enabled = 1;
			for (i = 0; i < NCPU; i++)
				if (IS_CPU(i))
					ftrace_cpu_stop(i);
			ftrace_state &= ~(FTRACE_ENABLED);
		}
		mutex_exit(&ftrace_lock);
	}
	return (was_enabled);
}

/*
 * ftrace_X() functions are called from trace context. All callers of ftrace_X()
 * tests FTRACE_ENABLED first. Although this is not very accurate, it keeps the
 * overhead very low when tracing is not enabled.
 *
 * gethrtime_unscaled() appears to be safe to be called in trace context. As an
 * added precaution, we call these before we disable interrupts on this cpu.
 */

void
ftrace_0(char *str, caddr_t caller)
{
	ftrace_record_t *r;
	struct cpu *cp;
	ftrace_data_t *ftd;
	ftrace_icookie_t cookie;
	hrtime_t  timestamp;

	timestamp = gethrtime_unscaled();

	cookie = ftrace_interrupt_disable();

	cp = CPU;
	ftd = &cp->cpu_ftrace;

	if (!(ftd->ftd_state & FTRACE_ENABLED)) {
		ftrace_interrupt_enable(cookie);
		return;
	}

	r = ftd->ftd_cur;
	r->ftr_event = str;
	r->ftr_thread = curthread;
	r->ftr_tick = timestamp;
	r->ftr_caller = caller;

	if (r++ == ftd->ftd_last)
		r = ftd->ftd_first;
	ftd->ftd_cur = r;

	ftrace_interrupt_enable(cookie);
}

void
ftrace_1(char *str, ulong_t arg1, caddr_t caller)
{
	ftrace_record_t *r;
	struct cpu *cp;
	ftrace_data_t *ftd;
	ftrace_icookie_t cookie;
	hrtime_t  timestamp;

	timestamp = gethrtime_unscaled();

	cookie = ftrace_interrupt_disable();

	cp = CPU;
	ftd = &cp->cpu_ftrace;

	if (!(ftd->ftd_state & FTRACE_ENABLED)) {
		ftrace_interrupt_enable(cookie);
		return;
	}

	r = ftd->ftd_cur;
	r->ftr_event = str;
	r->ftr_thread = curthread;
	r->ftr_tick = timestamp;
	r->ftr_caller = caller;
	r->ftr_data1 = arg1;

	if (r++ == ftd->ftd_last)
		r = ftd->ftd_first;
	ftd->ftd_cur = r;

	ftrace_interrupt_enable(cookie);
}

void
ftrace_2(char *str, ulong_t arg1, ulong_t arg2, caddr_t caller)
{
	ftrace_record_t *r;
	struct cpu *cp;
	ftrace_data_t *ftd;
	ftrace_icookie_t cookie;
	hrtime_t  timestamp;

	timestamp = gethrtime_unscaled();

	cookie = ftrace_interrupt_disable();

	cp = CPU;
	ftd = &cp->cpu_ftrace;

	if (!(ftd->ftd_state & FTRACE_ENABLED)) {
		ftrace_interrupt_enable(cookie);
		return;
	}

	r = ftd->ftd_cur;
	r->ftr_event = str;
	r->ftr_thread = curthread;
	r->ftr_tick = timestamp;
	r->ftr_caller = caller;
	r->ftr_data1 = arg1;
	r->ftr_data2 = arg2;

	if (r++ == ftd->ftd_last)
		r = ftd->ftd_first;
	ftd->ftd_cur = r;

	ftrace_interrupt_enable(cookie);
}

void
ftrace_3(char *str, ulong_t arg1, ulong_t arg2, ulong_t arg3, caddr_t caller)
{
	ftrace_record_t *r;
	struct cpu *cp;
	ftrace_data_t *ftd;
	ftrace_icookie_t cookie;
	hrtime_t  timestamp;

	timestamp = gethrtime_unscaled();

	cookie = ftrace_interrupt_disable();

	cp = CPU;
	ftd = &cp->cpu_ftrace;

	if (!(ftd->ftd_state & FTRACE_ENABLED)) {
		ftrace_interrupt_enable(cookie);
		return;
	}

	r = ftd->ftd_cur;
	r->ftr_event = str;
	r->ftr_thread = curthread;
	r->ftr_tick = timestamp;
	r->ftr_caller = caller;
	r->ftr_data1 = arg1;
	r->ftr_data2 = arg2;
	r->ftr_data3 = arg3;

	if (r++ == ftd->ftd_last)
		r = ftd->ftd_first;
	ftd->ftd_cur = r;

	ftrace_interrupt_enable(cookie);
}

void
ftrace_3_notick(char *str, ulong_t arg1, ulong_t arg2,
    ulong_t arg3, caddr_t caller)
{
	ftrace_record_t *r;
	struct cpu *cp;
	ftrace_data_t *ftd;
	ftrace_icookie_t cookie;

	cookie = ftrace_interrupt_disable();

	cp = CPU;
	ftd = &cp->cpu_ftrace;

	if (!(ftd->ftd_state & FTRACE_ENABLED)) {
		ftrace_interrupt_enable(cookie);
		return;
	}

	r = ftd->ftd_cur;
	r->ftr_event = str;
	r->ftr_thread = curthread;
	r->ftr_tick = 0;
	r->ftr_caller = caller;
	r->ftr_data1 = arg1;
	r->ftr_data2 = arg2;
	r->ftr_data3 = arg3;

	if (r++ == ftd->ftd_last)
		r = ftd->ftd_first;
	ftd->ftd_cur = r;

	ftrace_interrupt_enable(cookie);
}