trace.c

// SPDX-License-Identifier: GPL-2.0
/*
 * ring buffer based function tracer
 *
 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally taken from the RT patch by:
 *    Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code from the latency_tracer, that is:
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 Nadia Yvette Chambers
 */
#include <linux/ring_buffer.h>
#include <linux/utsname.h>
#include <linux/stacktrace.h>
#include <linux/writeback.h>
#include <linux/kallsyms.h>
#include <linux/security.h>
#include <linux/seq_file.h>
#include <linux/irqflags.h>
#include <linux/debugfs.h>
#include <linux/tracefs.h>
#include <linux/pagemap.h>
#include <linux/hardirq.h>
#include <linux/linkage.h>
#include <linux/uaccess.h>
#include <linux/cleanup.h>
#include <linux/vmalloc.h>
#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/splice.h>
#include <linux/kdebug.h>
#include <linux/string.h>
#include <linux/mount.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
#include <linux/ctype.h>
#include <linux/init.h>
#include <linux/panic_notifier.h>
#include <linux/poll.h>
#include <linux/nmi.h>
#include <linux/fs.h>
#include <linux/trace.h>
#include <linux/sched/clock.h>
#include <linux/sched/rt.h>
#include <linux/fsnotify.h>
#include <linux/irq_work.h>
#include <linux/workqueue.h>
#include <linux/sort.h>
#include <linux/io.h> /* vmap_page_range() */
#include <linux/fs_context.h>

#include <asm/setup.h> /* COMMAND_LINE_SIZE */

#include "trace.h"
#include "trace_output.h"

#ifdef CONFIG_FTRACE_STARTUP_TEST
/*
 * We need to change this state when a selftest is running.
 * A selftest will lurk into the ring-buffer to count the
 * entries inserted during the selftest although some concurrent
 * insertions into the ring-buffer such as trace_printk could occurred
 * at the same time, giving false positive or negative results.
 */
static bool __read_mostly tracing_selftest_running;

/*
 * If boot-time tracing including tracers/events via kernel cmdline
 * is running, we do not want to run SELFTEST.
 */
bool __read_mostly tracing_selftest_disabled;

void __init disable_tracing_selftest(const char *reason)
{
	if (!tracing_selftest_disabled) {
		tracing_selftest_disabled = true;
		pr_info("Ftrace startup test is disabled due to %s\n", reason);
	}
}
#else
#define tracing_selftest_running	0
#define tracing_selftest_disabled	0
#endif

/* Pipe tracepoints to printk */
static struct trace_iterator *tracepoint_print_iter;
int tracepoint_printk;
static bool tracepoint_printk_stop_on_boot __initdata;
static bool traceoff_after_boot __initdata;
static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);

/* For tracers that don't implement custom flags */
static struct tracer_opt dummy_tracer_opt[] = {
	{ }
};

static int
dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
	return 0;
}

/*
 * To prevent the comm cache from being overwritten when no
 * tracing is active, only save the comm when a trace event
 * occurred.
 */
DEFINE_PER_CPU(bool, trace_taskinfo_save);

/*
 * Kill all tracing for good (never come back).
 * It is initialized to 1 but will turn to zero if the initialization
 * of the tracer is successful. But that is the only place that sets
 * this back to zero.
 */
static int tracing_disabled = 1;

cpumask_var_t __read_mostly	tracing_buffer_mask;

#define MAX_TRACER_SIZE		100
/*
 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
 *
 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
 * is set, then ftrace_dump is called. This will output the contents
 * of the ftrace buffers to the console.  This is very useful for
 * capturing traces that lead to crashes and outputing it to a
 * serial console.
 *
 * It is default off, but you can enable it with either specifying
 * "ftrace_dump_on_oops" in the kernel command line, or setting
 * /proc/sys/kernel/ftrace_dump_on_oops
 * Set 1 if you want to dump buffers of all CPUs
 * Set 2 if you want to dump the buffer of the CPU that triggered oops
 * Set instance name if you want to dump the specific trace instance
 * Multiple instance dump is also supported, and instances are seperated
 * by commas.
 */
/* Set to string format zero to disable by default */
char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0";

/* When set, tracing will stop when a WARN*() is hit */
static int __disable_trace_on_warning;

int tracepoint_printk_sysctl(const struct ctl_table *table, int write,
			     void *buffer, size_t *lenp, loff_t *ppos);
static const struct ctl_table trace_sysctl_table[] = {
	{
		.procname	= "ftrace_dump_on_oops",
		.data		= &ftrace_dump_on_oops,
		.maxlen		= MAX_TRACER_SIZE,
		.mode		= 0644,
		.proc_handler	= proc_dostring,
	},
	{
		.procname	= "traceoff_on_warning",
		.data		= &__disable_trace_on_warning,
		.maxlen		= sizeof(__disable_trace_on_warning),
		.mode		= 0644,
		.proc_handler	= proc_dointvec,
	},
	{
		.procname	= "tracepoint_printk",
		.data		= &tracepoint_printk,
		.maxlen		= sizeof(tracepoint_printk),
		.mode		= 0644,
		.proc_handler	= tracepoint_printk_sysctl,
	},
};

static int __init init_trace_sysctls(void)
{
	register_sysctl_init("kernel", trace_sysctl_table);
	return 0;
}
subsys_initcall(init_trace_sysctls);

#ifdef CONFIG_TRACE_EVAL_MAP_FILE
/* Map of enums to their values, for "eval_map" file */
struct trace_eval_map_head {
	struct module			*mod;
	unsigned long			length;
};

union trace_eval_map_item;

struct trace_eval_map_tail {
	/*
	 * "end" is first and points to NULL as it must be different
	 * than "mod" or "eval_string"
	 */
	union trace_eval_map_item	*next;
	const char			*end;	/* points to NULL */
};

static DEFINE_MUTEX(trace_eval_mutex);

/*
 * The trace_eval_maps are saved in an array with two extra elements,
 * one at the beginning, and one at the end. The beginning item contains
 * the count of the saved maps (head.length), and the module they
 * belong to if not built in (head.mod). The ending item contains a
 * pointer to the next array of saved eval_map items.
 */
union trace_eval_map_item {
	struct trace_eval_map		map;
	struct trace_eval_map_head	head;
	struct trace_eval_map_tail	tail;
};

static union trace_eval_map_item *trace_eval_maps;
#endif /* CONFIG_TRACE_EVAL_MAP_FILE */

int tracing_set_tracer(struct trace_array *tr, const char *buf);
static void ftrace_trace_userstack(struct trace_array *tr,
				   struct trace_buffer *buffer,
				   unsigned int trace_ctx);

static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
static char *default_bootup_tracer;

static bool allocate_snapshot;
static bool snapshot_at_boot;

static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
static int boot_instance_index;

static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata;
static int boot_snapshot_index;

static int __init set_cmdline_ftrace(char *str)
{
	strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
	default_bootup_tracer = bootup_tracer_buf;
	/* We are using ftrace early, expand it */
	trace_set_ring_buffer_expanded(NULL);
	return 1;
}
__setup("ftrace=", set_cmdline_ftrace);

int ftrace_dump_on_oops_enabled(void)
{
	if (!strcmp("0", ftrace_dump_on_oops))
		return 0;
	else
		return 1;
}

static int __init set_ftrace_dump_on_oops(char *str)
{
	if (!*str) {
		strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
		return 1;
	}

	if (*str == ',') {
		strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
		strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);
		return 1;
	}

	if (*str++ == '=') {
		strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE);
		return 1;
	}

	return 0;
}
__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);

static int __init stop_trace_on_warning(char *str)
{
	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
		__disable_trace_on_warning = 1;
	return 1;
}
__setup("traceoff_on_warning", stop_trace_on_warning);

static int __init boot_alloc_snapshot(char *str)
{
	char *slot = boot_snapshot_info + boot_snapshot_index;
	int left = sizeof(boot_snapshot_info) - boot_snapshot_index;
	int ret;

	if (str[0] == '=') {
		str++;
		if (strlen(str) >= left)
			return -1;

		ret = snprintf(slot, left, "%s\t", str);
		boot_snapshot_index += ret;
	} else {
		allocate_snapshot = true;
		/* We also need the main ring buffer expanded */
		trace_set_ring_buffer_expanded(NULL);
	}
	return 1;
}
__setup("alloc_snapshot", boot_alloc_snapshot);


static int __init boot_snapshot(char *str)
{
	snapshot_at_boot = true;
	boot_alloc_snapshot(str);
	return 1;
}
__setup("ftrace_boot_snapshot", boot_snapshot);


static int __init boot_instance(char *str)
{
	char *slot = boot_instance_info + boot_instance_index;
	int left = sizeof(boot_instance_info) - boot_instance_index;
	int ret;

	if (strlen(str) >= left)
		return -1;

	ret = snprintf(slot, left, "%s\t", str);
	boot_instance_index += ret;

	return 1;
}
__setup("trace_instance=", boot_instance);


static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;

static int __init set_trace_boot_options(char *str)
{
	strscpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
	return 1;
}
__setup("trace_options=", set_trace_boot_options);

static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
static char *trace_boot_clock __initdata;

static int __init set_trace_boot_clock(char *str)
{
	strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
	trace_boot_clock = trace_boot_clock_buf;
	return 1;
}
__setup("trace_clock=", set_trace_boot_clock);

static int __init set_tracepoint_printk(char *str)
{
	/* Ignore the "tp_printk_stop_on_boot" param */
	if (*str == '_')
		return 0;

	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
		tracepoint_printk = 1;
	return 1;
}
__setup("tp_printk", set_tracepoint_printk);

static int __init set_tracepoint_printk_stop(char *str)
{
	tracepoint_printk_stop_on_boot = true;
	return 1;
}
__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);

static int __init set_traceoff_after_boot(char *str)
{
	traceoff_after_boot = true;
	return 1;
}
__setup("traceoff_after_boot", set_traceoff_after_boot);

unsigned long long ns2usecs(u64 nsec)
{
	nsec += 500;
	do_div(nsec, 1000);
	return nsec;
}

static void
trace_process_export(struct trace_export *export,
	       struct ring_buffer_event *event, int flag)
{
	struct trace_entry *entry;
	unsigned int size = 0;

	if (export->flags & flag) {
		entry = ring_buffer_event_data(event);
		size = ring_buffer_event_length(event);
		export->write(export, entry, size);
	}
}

static DEFINE_MUTEX(ftrace_export_lock);

static struct trace_export __rcu *ftrace_exports_list __read_mostly;

static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);

static inline void ftrace_exports_enable(struct trace_export *export)
{
	if (export->flags & TRACE_EXPORT_FUNCTION)
		static_branch_inc(&trace_function_exports_enabled);

	if (export->flags & TRACE_EXPORT_EVENT)
		static_branch_inc(&trace_event_exports_enabled);

	if (export->flags & TRACE_EXPORT_MARKER)
		static_branch_inc(&trace_marker_exports_enabled);
}

static inline void ftrace_exports_disable(struct trace_export *export)
{
	if (export->flags & TRACE_EXPORT_FUNCTION)
		static_branch_dec(&trace_function_exports_enabled);

	if (export->flags & TRACE_EXPORT_EVENT)
		static_branch_dec(&trace_event_exports_enabled);

	if (export->flags & TRACE_EXPORT_MARKER)
		static_branch_dec(&trace_marker_exports_enabled);
}

static void ftrace_exports(struct ring_buffer_event *event, int flag)
{
	struct trace_export *export;

	guard(preempt_notrace)();

	export = rcu_dereference_raw_check(ftrace_exports_list);
	while (export) {
		trace_process_export(export, event, flag);
		export = rcu_dereference_raw_check(export->next);
	}
}

static inline void
add_trace_export(struct trace_export **list, struct trace_export *export)
{
	rcu_assign_pointer(export->next, *list);
	/*
	 * We are entering export into the list but another
	 * CPU might be walking that list. We need to make sure
	 * the export->next pointer is valid before another CPU sees
	 * the export pointer included into the list.
	 */
	rcu_assign_pointer(*list, export);
}

static inline int
rm_trace_export(struct trace_export **list, struct trace_export *export)
{
	struct trace_export **p;

	for (p = list; *p != NULL; p = &(*p)->next)
		if (*p == export)
			break;

	if (*p != export)
		return -1;

	rcu_assign_pointer(*p, (*p)->next);

	return 0;
}

static inline void
add_ftrace_export(struct trace_export **list, struct trace_export *export)
{
	ftrace_exports_enable(export);

	add_trace_export(list, export);
}

static inline int
rm_ftrace_export(struct trace_export **list, struct trace_export *export)
{
	int ret;

	ret = rm_trace_export(list, export);
	ftrace_exports_disable(export);

	return ret;
}

int register_ftrace_export(struct trace_export *export)
{
	if (WARN_ON_ONCE(!export->write))
		return -1;

	guard(mutex)(&ftrace_export_lock);

	add_ftrace_export(&ftrace_exports_list, export);

	return 0;
}
EXPORT_SYMBOL_GPL(register_ftrace_export);

int unregister_ftrace_export(struct trace_export *export)
{
	guard(mutex)(&ftrace_export_lock);
	return rm_ftrace_export(&ftrace_exports_list, export);
}
EXPORT_SYMBOL_GPL(unregister_ftrace_export);

/* trace_flags holds trace_options default values */
#define TRACE_DEFAULT_FLAGS						\
	(FUNCTION_DEFAULT_FLAGS |					\
	 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |			\
	 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO |		\
	 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |			\
	 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS |			\
	 TRACE_ITER_HASH_PTR | TRACE_ITER_TRACE_PRINTK |		\
	 TRACE_ITER_COPY_MARKER)

/* trace_options that are only supported by global_trace */
#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK |			\
	       TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)

/* trace_flags that are default zero for instances */
#define ZEROED_TRACE_FLAGS \
	(TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK | TRACE_ITER_TRACE_PRINTK | \
	 TRACE_ITER_COPY_MARKER)

/*
 * The global_trace is the descriptor that holds the top-level tracing
 * buffers for the live tracing.
 */
static struct trace_array global_trace = {
	.trace_flags = TRACE_DEFAULT_FLAGS,
};

static struct trace_array *printk_trace = &global_trace;

/* List of trace_arrays interested in the top level trace_marker */
static LIST_HEAD(marker_copies);

static __always_inline bool printk_binsafe(struct trace_array *tr)
{
	/*
	 * The binary format of traceprintk can cause a crash if used
	 * by a buffer from another boot. Force the use of the
	 * non binary version of trace_printk if the trace_printk
	 * buffer is a boot mapped ring buffer.
	 */
	return !(tr->flags & TRACE_ARRAY_FL_BOOT);
}

static void update_printk_trace(struct trace_array *tr)
{
	if (printk_trace == tr)
		return;

	printk_trace->trace_flags &= ~TRACE_ITER_TRACE_PRINTK;
	printk_trace = tr;
	tr->trace_flags |= TRACE_ITER_TRACE_PRINTK;
}

/* Returns true if the status of tr changed */
static bool update_marker_trace(struct trace_array *tr, int enabled)
{
	lockdep_assert_held(&event_mutex);

	if (enabled) {
		if (!list_empty(&tr->marker_list))
			return false;

		list_add_rcu(&tr->marker_list, &marker_copies);
		tr->trace_flags |= TRACE_ITER_COPY_MARKER;
		return true;
	}

	if (list_empty(&tr->marker_list))
		return false;

	list_del_init(&tr->marker_list);
	tr->trace_flags &= ~TRACE_ITER_COPY_MARKER;
	return true;
}

void trace_set_ring_buffer_expanded(struct trace_array *tr)
{
	if (!tr)
		tr = &global_trace;
	tr->ring_buffer_expanded = true;
}

LIST_HEAD(ftrace_trace_arrays);

int trace_array_get(struct trace_array *this_tr)
{
	struct trace_array *tr;

	guard(mutex)(&trace_types_lock);
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (tr == this_tr) {
			tr->ref++;
			return 0;
		}
	}

	return -ENODEV;
}

static void __trace_array_put(struct trace_array *this_tr)
{
	WARN_ON(!this_tr->ref);
	this_tr->ref--;
}

/**
 * trace_array_put - Decrement the reference counter for this trace array.
 * @this_tr : pointer to the trace array
 *
 * NOTE: Use this when we no longer need the trace array returned by
 * trace_array_get_by_name(). This ensures the trace array can be later
 * destroyed.
 *
 */
void trace_array_put(struct trace_array *this_tr)
{
	if (!this_tr)
		return;

	guard(mutex)(&trace_types_lock);
	__trace_array_put(this_tr);
}
EXPORT_SYMBOL_GPL(trace_array_put);

int tracing_check_open_get_tr(struct trace_array *tr)
{
	int ret;

	ret = security_locked_down(LOCKDOWN_TRACEFS);
	if (ret)
		return ret;

	if (tracing_disabled)
		return -ENODEV;

	if (tr && trace_array_get(tr) < 0)
		return -ENODEV;

	return 0;
}

/**
 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
 * @filtered_pids: The list of pids to check
 * @search_pid: The PID to find in @filtered_pids
 *
 * Returns true if @search_pid is found in @filtered_pids, and false otherwise.
 */
bool
trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
{
	return trace_pid_list_is_set(filtered_pids, search_pid);
}

/**
 * trace_ignore_this_task - should a task be ignored for tracing
 * @filtered_pids: The list of pids to check
 * @filtered_no_pids: The list of pids not to be traced
 * @task: The task that should be ignored if not filtered
 *
 * Checks if @task should be traced or not from @filtered_pids.
 * Returns true if @task should *NOT* be traced.
 * Returns false if @task should be traced.
 */
bool
trace_ignore_this_task(struct trace_pid_list *filtered_pids,
		       struct trace_pid_list *filtered_no_pids,
		       struct task_struct *task)
{
	/*
	 * If filtered_no_pids is not empty, and the task's pid is listed
	 * in filtered_no_pids, then return true.
	 * Otherwise, if filtered_pids is empty, that means we can
	 * trace all tasks. If it has content, then only trace pids
	 * within filtered_pids.
	 */

	return (filtered_pids &&
		!trace_find_filtered_pid(filtered_pids, task->pid)) ||
		(filtered_no_pids &&
		 trace_find_filtered_pid(filtered_no_pids, task->pid));
}

/**
 * trace_filter_add_remove_task - Add or remove a task from a pid_list
 * @pid_list: The list to modify
 * @self: The current task for fork or NULL for exit
 * @task: The task to add or remove
 *
 * If adding a task, if @self is defined, the task is only added if @self
 * is also included in @pid_list. This happens on fork and tasks should
 * only be added when the parent is listed. If @self is NULL, then the
 * @task pid will be removed from the list, which would happen on exit
 * of a task.
 */
void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
				  struct task_struct *self,
				  struct task_struct *task)
{
	if (!pid_list)
		return;

	/* For forks, we only add if the forking task is listed */
	if (self) {
		if (!trace_find_filtered_pid(pid_list, self->pid))
			return;
	}

	/* "self" is set for forks, and NULL for exits */
	if (self)
		trace_pid_list_set(pid_list, task->pid);
	else
		trace_pid_list_clear(pid_list, task->pid);
}

/**
 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
 * @pid_list: The pid list to show
 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
 * @pos: The position of the file
 *
 * This is used by the seq_file "next" operation to iterate the pids
 * listed in a trace_pid_list structure.
 *
 * Returns the pid+1 as we want to display pid of zero, but NULL would
 * stop the iteration.
 */
void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
{
	long pid = (unsigned long)v;
	unsigned int next;

	(*pos)++;

	/* pid already is +1 of the actual previous bit */
	if (trace_pid_list_next(pid_list, pid, &next) < 0)
		return NULL;

	pid = next;

	/* Return pid + 1 to allow zero to be represented */
	return (void *)(pid + 1);
}

/**
 * trace_pid_start - Used for seq_file to start reading pid lists
 * @pid_list: The pid list to show
 * @pos: The position of the file
 *
 * This is used by seq_file "start" operation to start the iteration
 * of listing pids.
 *
 * Returns the pid+1 as we want to display pid of zero, but NULL would
 * stop the iteration.
 */
void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
{
	unsigned long pid;
	unsigned int first;
	loff_t l = 0;

	if (trace_pid_list_first(pid_list, &first) < 0)
		return NULL;

	pid = first;

	/* Return pid + 1 so that zero can be the exit value */
	for (pid++; pid && l < *pos;
	     pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
		;
	return (void *)pid;
}

/**
 * trace_pid_show - show the current pid in seq_file processing
 * @m: The seq_file structure to write into
 * @v: A void pointer of the pid (+1) value to display
 *
 * Can be directly used by seq_file operations to display the current
 * pid value.
 */
int trace_pid_show(struct seq_file *m, void *v)
{
	unsigned long pid = (unsigned long)v - 1;

	seq_printf(m, "%lu\n", pid);
	return 0;
}

/* 128 should be much more than enough */
#define PID_BUF_SIZE		127

int trace_pid_write(struct trace_pid_list *filtered_pids,
		    struct trace_pid_list **new_pid_list,
		    const char __user *ubuf, size_t cnt)
{
	struct trace_pid_list *pid_list;
	struct trace_parser parser;
	unsigned long val;
	int nr_pids = 0;
	ssize_t read = 0;
	ssize_t ret;
	loff_t pos;
	pid_t pid;

	if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
		return -ENOMEM;

	/*
	 * Always recreate a new array. The write is an all or nothing
	 * operation. Always create a new array when adding new pids by
	 * the user. If the operation fails, then the current list is
	 * not modified.
	 */
	pid_list = trace_pid_list_alloc();
	if (!pid_list) {
		trace_parser_put(&parser);
		return -ENOMEM;
	}

	if (filtered_pids) {
		/* copy the current bits to the new max */
		ret = trace_pid_list_first(filtered_pids, &pid);
		while (!ret) {
			ret = trace_pid_list_set(pid_list, pid);
			if (ret < 0)
				goto out;

			ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
			nr_pids++;
		}
	}

	ret = 0;
	while (cnt > 0) {

		pos = 0;

		ret = trace_get_user(&parser, ubuf, cnt, &pos);
		if (ret < 0)
			break;

		read += ret;
		ubuf += ret;
		cnt -= ret;

		if (!trace_parser_loaded(&parser))
			break;

		ret = -EINVAL;
		if (kstrtoul(parser.buffer, 0, &val))
			break;

		pid = (pid_t)val;

		if (trace_pid_list_set(pid_list, pid) < 0) {
			ret = -1;
			break;
		}
		nr_pids++;

		trace_parser_clear(&parser);
		ret = 0;
	}
 out:
	trace_parser_put(&parser);

	if (ret < 0) {
		trace_pid_list_free(pid_list);
		return ret;
	}

	if (!nr_pids) {
		/* Cleared the list of pids */
		trace_pid_list_free(pid_list);
		pid_list = NULL;
	}

	*new_pid_list = pid_list;

	return read;
}

static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
{
	u64 ts;

	/* Early boot up does not have a buffer yet */
	if (!buf->buffer)
		return trace_clock_local();

	ts = ring_buffer_time_stamp(buf->buffer);
	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);

	return ts;
}

u64 ftrace_now(int cpu)
{
	return buffer_ftrace_now(&global_trace.array_buffer, cpu);
}

/**
 * tracing_is_enabled - Show if global_trace has been enabled
 *
 * Shows if the global trace has been enabled or not. It uses the
 * mirror flag "buffer_disabled" to be used in fast paths such as for
 * the irqsoff tracer. But it may be inaccurate due to races. If you
 * need to know the accurate state, use tracing_is_on() which is a little
 * slower, but accurate.
 */
int tracing_is_enabled(void)
{
	/*
	 * For quick access (irqsoff uses this in fast path), just
	 * return the mirror variable of the state of the ring buffer.
	 * It's a little racy, but we don't really care.
	 */
	return !global_trace.buffer_disabled;
}

/*
 * trace_buf_size is the size in bytes that is allocated
 * for a buffer. Note, the number of bytes is always rounded
 * to page size.
 *
 * This number is purposely set to a low number of 16384.
 * If the dump on oops happens, it will be much appreciated
 * to not have to wait for all that output. Anyway this can be
 * boot time and run time configurable.
 */
#define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */

static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;

/* trace_types holds a link list of available tracers. */
static struct tracer		*trace_types __read_mostly;

/*
 * trace_types_lock is used to protect the trace_types list.
 */
DEFINE_MUTEX(trace_types_lock);

/*
 * serialize the access of the ring buffer
 *
 * ring buffer serializes readers, but it is low level protection.
 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 * are not protected by ring buffer.
 *
 * The content of events may become garbage if we allow other process consumes
 * these events concurrently:
 *   A) the page of the consumed events may become a normal page
 *      (not reader page) in ring buffer, and this page will be rewritten
 *      by events producer.
 *   B) The page of the consumed events may become a page for splice_read,
 *      and this page will be returned to system.
 *
 * These primitives allow multi process access to different cpu ring buffer
 * concurrently.
 *
 * These primitives don't distinguish read-only and read-consume access.
 * Multi read-only access are also serialized.
 */

#ifdef CONFIG_SMP
static DECLARE_RWSEM(all_cpu_access_lock);
static DEFINE_PER_CPU(struct mutex, cpu_access_lock);

static inline void trace_access_lock(int cpu)
{
	if (cpu == RING_BUFFER_ALL_CPUS) {
		/* gain it for accessing the whole ring buffer. */
		down_write(&all_cpu_access_lock);
	} else {
		/* gain it for accessing a cpu ring buffer. */

		/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
		down_read(&all_cpu_access_lock);

		/* Secondly block other access to this @cpu ring buffer. */
		mutex_lock(&per_cpu(cpu_access_lock, cpu));
	}
}

static inline void trace_access_unlock(int cpu)
{
	if (cpu == RING_BUFFER_ALL_CPUS) {