syscall.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 */
#include <crypto/sha2.h>
#include <linux/bpf.h>
#include <linux/bpf-cgroup.h>
#include <linux/bpf_trace.h>
#include <linux/bpf_lirc.h>
#include <linux/bpf_verifier.h>
#include <linux/bsearch.h>
#include <linux/btf.h>
#include <linux/syscalls.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/vmalloc.h>
#include <linux/mmzone.h>
#include <linux/anon_inodes.h>
#include <linux/fdtable.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/license.h>
#include <linux/filter.h>
#include <linux/kernel.h>
#include <linux/idr.h>
#include <linux/cred.h>
#include <linux/timekeeping.h>
#include <linux/ctype.h>
#include <linux/nospec.h>
#include <linux/audit.h>
#include <uapi/linux/btf.h>
#include <linux/pgtable.h>
#include <linux/bpf_lsm.h>
#include <linux/poll.h>
#include <linux/sort.h>
#include <linux/bpf-netns.h>
#include <linux/rcupdate_trace.h>
#include <linux/memcontrol.h>
#include <linux/trace_events.h>
#include <linux/tracepoint.h>
#include <linux/overflow.h>
#include <linux/cookie.h>
#include <linux/verification.h>

#include <net/netfilter/nf_bpf_link.h>
#include <net/netkit.h>
#include <net/tcx.h>

#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
			  (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
			  (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
#define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
			IS_FD_HASH(map))

#define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)

DEFINE_PER_CPU(int, bpf_prog_active);
DEFINE_COOKIE(bpf_map_cookie);
static DEFINE_IDR(prog_idr);
static DEFINE_SPINLOCK(prog_idr_lock);
static DEFINE_IDR(map_idr);
static DEFINE_SPINLOCK(map_idr_lock);
static DEFINE_IDR(link_idr);
static DEFINE_SPINLOCK(link_idr_lock);

int sysctl_unprivileged_bpf_disabled __read_mostly =
	IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;

static const struct bpf_map_ops * const bpf_map_types[] = {
#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
#define BPF_MAP_TYPE(_id, _ops) \
	[_id] = &_ops,
#define BPF_LINK_TYPE(_id, _name)
#include <linux/bpf_types.h>
#undef BPF_PROG_TYPE
#undef BPF_MAP_TYPE
#undef BPF_LINK_TYPE
};

/*
 * If we're handed a bigger struct than we know of, ensure all the unknown bits
 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
 * we don't know about yet.
 *
 * There is a ToCToU between this function call and the following
 * copy_from_user() call. However, this is not a concern since this function is
 * meant to be a future-proofing of bits.
 */
int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
			     size_t expected_size,
			     size_t actual_size)
{
	int res;

	if (unlikely(actual_size > PAGE_SIZE))	/* silly large */
		return -E2BIG;

	if (actual_size <= expected_size)
		return 0;

	if (uaddr.is_kernel)
		res = memchr_inv(uaddr.kernel + expected_size, 0,
				 actual_size - expected_size) == NULL;
	else
		res = check_zeroed_user(uaddr.user + expected_size,
					actual_size - expected_size);
	if (res < 0)
		return res;
	return res ? 0 : -E2BIG;
}

const struct bpf_map_ops bpf_map_offload_ops = {
	.map_meta_equal = bpf_map_meta_equal,
	.map_alloc = bpf_map_offload_map_alloc,
	.map_free = bpf_map_offload_map_free,
	.map_check_btf = map_check_no_btf,
	.map_mem_usage = bpf_map_offload_map_mem_usage,
};

static void bpf_map_write_active_inc(struct bpf_map *map)
{
	atomic64_inc(&map->writecnt);
}

static void bpf_map_write_active_dec(struct bpf_map *map)
{
	atomic64_dec(&map->writecnt);
}

bool bpf_map_write_active(const struct bpf_map *map)
{
	return atomic64_read(&map->writecnt) != 0;
}

static u32 bpf_map_value_size(const struct bpf_map *map)
{
	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
	    map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
		return round_up(map->value_size, 8) * num_possible_cpus();
	else if (IS_FD_MAP(map))
		return sizeof(u32);
	else
		return  map->value_size;
}

static void maybe_wait_bpf_programs(struct bpf_map *map)
{
	/* Wait for any running non-sleepable BPF programs to complete so that
	 * userspace, when we return to it, knows that all non-sleepable
	 * programs that could be running use the new map value. For sleepable
	 * BPF programs, synchronize_rcu_tasks_trace() should be used to wait
	 * for the completions of these programs, but considering the waiting
	 * time can be very long and userspace may think it will hang forever,
	 * so don't handle sleepable BPF programs now.
	 */
	if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
	    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
		synchronize_rcu();
}

static void unpin_uptr_kaddr(void *kaddr)
{
	if (kaddr)
		unpin_user_page(virt_to_page(kaddr));
}

static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)
{
	const struct btf_field *field;
	void **uptr_addr;
	int i;

	for (i = 0, field = rec->fields; i < cnt; i++, field++) {
		if (field->type != BPF_UPTR)
			continue;

		uptr_addr = obj + field->offset;
		unpin_uptr_kaddr(*uptr_addr);
	}
}

static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)
{
	if (!btf_record_has_field(rec, BPF_UPTR))
		return;

	__bpf_obj_unpin_uptrs(rec, rec->cnt, obj);
}

static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)
{
	const struct btf_field *field;
	const struct btf_type *t;
	unsigned long start, end;
	struct page *page;
	void **uptr_addr;
	int i, err;

	if (!btf_record_has_field(rec, BPF_UPTR))
		return 0;

	for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
		if (field->type != BPF_UPTR)
			continue;

		uptr_addr = obj + field->offset;
		start = *(unsigned long *)uptr_addr;
		if (!start)
			continue;

		t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
		/* t->size was checked for zero before */
		if (check_add_overflow(start, t->size - 1, &end)) {
			err = -EFAULT;
			goto unpin_all;
		}

		/* The uptr's struct cannot span across two pages */
		if ((start & PAGE_MASK) != (end & PAGE_MASK)) {
			err = -EOPNOTSUPP;
			goto unpin_all;
		}

		err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);
		if (err != 1)
			goto unpin_all;

		if (PageHighMem(page)) {
			err = -EOPNOTSUPP;
			unpin_user_page(page);
			goto unpin_all;
		}

		*uptr_addr = page_address(page) + offset_in_page(start);
	}

	return 0;

unpin_all:
	__bpf_obj_unpin_uptrs(rec, i, obj);
	return err;
}

static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
				void *key, void *value, __u64 flags)
{
	int err;

	/* Need to create a kthread, thus must support schedule */
	if (bpf_map_is_offloaded(map)) {
		return bpf_map_offload_update_elem(map, key, value, flags);
	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
		   map->map_type == BPF_MAP_TYPE_ARENA ||
		   map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
		return map->ops->map_update_elem(map, key, value, flags);
	} else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
		   map->map_type == BPF_MAP_TYPE_SOCKMAP) {
		return sock_map_update_elem_sys(map, key, value, flags);
	} else if (IS_FD_PROG_ARRAY(map)) {
		return bpf_fd_array_map_update_elem(map, map_file, key, value,
						    flags);
	}

	bpf_disable_instrumentation();
	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
		err = bpf_percpu_hash_update(map, key, value, flags);
	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
		err = bpf_percpu_array_update(map, key, value, flags);
	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
		err = bpf_percpu_cgroup_storage_update(map, key, value,
						       flags);
	} else if (IS_FD_ARRAY(map)) {
		err = bpf_fd_array_map_update_elem(map, map_file, key, value,
						   flags);
	} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
		err = bpf_fd_htab_map_update_elem(map, map_file, key, value,
						  flags);
	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
		/* rcu_read_lock() is not needed */
		err = bpf_fd_reuseport_array_update_elem(map, key, value,
							 flags);
	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
		   map->map_type == BPF_MAP_TYPE_STACK ||
		   map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
		err = map->ops->map_push_elem(map, value, flags);
	} else {
		err = bpf_obj_pin_uptrs(map->record, value);
		if (!err) {
			rcu_read_lock();
			err = map->ops->map_update_elem(map, key, value, flags);
			rcu_read_unlock();
			if (err)
				bpf_obj_unpin_uptrs(map->record, value);
		}
	}
	bpf_enable_instrumentation();

	return err;
}

static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
			      __u64 flags)
{
	void *ptr;
	int err;

	if (bpf_map_is_offloaded(map))
		return bpf_map_offload_lookup_elem(map, key, value);

	bpf_disable_instrumentation();
	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
		err = bpf_percpu_hash_copy(map, key, value);
	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
		err = bpf_percpu_array_copy(map, key, value);
	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
		err = bpf_percpu_cgroup_storage_copy(map, key, value);
	} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
		err = bpf_stackmap_extract(map, key, value, false);
	} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
		err = bpf_fd_array_map_lookup_elem(map, key, value);
	} else if (IS_FD_HASH(map)) {
		err = bpf_fd_htab_map_lookup_elem(map, key, value);
	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
		err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
		   map->map_type == BPF_MAP_TYPE_STACK ||
		   map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
		err = map->ops->map_peek_elem(map, value);
	} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
		/* struct_ops map requires directly updating "value" */
		err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
	} else {
		rcu_read_lock();
		if (map->ops->map_lookup_elem_sys_only)
			ptr = map->ops->map_lookup_elem_sys_only(map, key);
		else
			ptr = map->ops->map_lookup_elem(map, key);
		if (IS_ERR(ptr)) {
			err = PTR_ERR(ptr);
		} else if (!ptr) {
			err = -ENOENT;
		} else {
			err = 0;
			if (flags & BPF_F_LOCK)
				/* lock 'ptr' and copy everything but lock */
				copy_map_value_locked(map, value, ptr, true);
			else
				copy_map_value(map, value, ptr);
			/* mask lock and timer, since value wasn't zero inited */
			check_and_init_map_value(map, value);
		}
		rcu_read_unlock();
	}

	bpf_enable_instrumentation();

	return err;
}

/* Please, do not use this function outside from the map creation path
 * (e.g. in map update path) without taking care of setting the active
 * memory cgroup (see at bpf_map_kmalloc_node() for example).
 */
static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
{
	/* We really just want to fail instead of triggering OOM killer
	 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
	 * which is used for lower order allocation requests.
	 *
	 * It has been observed that higher order allocation requests done by
	 * vmalloc with __GFP_NORETRY being set might fail due to not trying
	 * to reclaim memory from the page cache, thus we set
	 * __GFP_RETRY_MAYFAIL to avoid such situations.
	 */

	gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);
	unsigned int flags = 0;
	unsigned long align = 1;
	void *area;

	if (size >= SIZE_MAX)
		return NULL;

	/* kmalloc()'ed memory can't be mmap()'ed */
	if (mmapable) {
		BUG_ON(!PAGE_ALIGNED(size));
		align = SHMLBA;
		flags = VM_USERMAP;
	} else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
		area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
				    numa_node);
		if (area != NULL)
			return area;
	}

	return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
			gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
			flags, numa_node, __builtin_return_address(0));
}

void *bpf_map_area_alloc(u64 size, int numa_node)
{
	return __bpf_map_area_alloc(size, numa_node, false);
}

void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
{
	return __bpf_map_area_alloc(size, numa_node, true);
}

void bpf_map_area_free(void *area)
{
	kvfree(area);
}

static u32 bpf_map_flags_retain_permanent(u32 flags)
{
	/* Some map creation flags are not tied to the map object but
	 * rather to the map fd instead, so they have no meaning upon
	 * map object inspection since multiple file descriptors with
	 * different (access) properties can exist here. Thus, given
	 * this has zero meaning for the map itself, lets clear these
	 * from here.
	 */
	return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
}

void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
{
	map->map_type = attr->map_type;
	map->key_size = attr->key_size;
	map->value_size = attr->value_size;
	map->max_entries = attr->max_entries;
	map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
	map->numa_node = bpf_map_attr_numa_node(attr);
	map->map_extra = attr->map_extra;
}

static int bpf_map_alloc_id(struct bpf_map *map)
{
	int id;

	idr_preload(GFP_KERNEL);
	spin_lock_bh(&map_idr_lock);
	id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
	if (id > 0)
		map->id = id;
	spin_unlock_bh(&map_idr_lock);
	idr_preload_end();

	if (WARN_ON_ONCE(!id))
		return -ENOSPC;

	return id > 0 ? 0 : id;
}

void bpf_map_free_id(struct bpf_map *map)
{
	unsigned long flags;

	/* Offloaded maps are removed from the IDR store when their device
	 * disappears - even if someone holds an fd to them they are unusable,
	 * the memory is gone, all ops will fail; they are simply waiting for
	 * refcnt to drop to be freed.
	 */
	if (!map->id)
		return;

	spin_lock_irqsave(&map_idr_lock, flags);

	idr_remove(&map_idr, map->id);
	map->id = 0;

	spin_unlock_irqrestore(&map_idr_lock, flags);
}

#ifdef CONFIG_MEMCG
static void bpf_map_save_memcg(struct bpf_map *map)
{
	/* Currently if a map is created by a process belonging to the root
	 * memory cgroup, get_obj_cgroup_from_current() will return NULL.
	 * So we have to check map->objcg for being NULL each time it's
	 * being used.
	 */
	if (memcg_bpf_enabled())
		map->objcg = get_obj_cgroup_from_current();
}

static void bpf_map_release_memcg(struct bpf_map *map)
{
	if (map->objcg)
		obj_cgroup_put(map->objcg);
}

static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)
{
	if (map->objcg)
		return get_mem_cgroup_from_objcg(map->objcg);

	return root_mem_cgroup;
}

void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
			   int node)
{
	struct mem_cgroup *memcg, *old_memcg;
	void *ptr;

	memcg = bpf_map_get_memcg(map);
	old_memcg = set_active_memcg(memcg);
	ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);

	return ptr;
}

void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
			     int node)
{
	struct mem_cgroup *memcg, *old_memcg;
	void *ptr;

	memcg = bpf_map_get_memcg(map);
	old_memcg = set_active_memcg(memcg);
	ptr = kmalloc_nolock(size, flags | __GFP_ACCOUNT, node);
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);

	return ptr;
}

void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
{
	struct mem_cgroup *memcg, *old_memcg;
	void *ptr;

	memcg = bpf_map_get_memcg(map);
	old_memcg = set_active_memcg(memcg);
	ptr = kzalloc(size, flags | __GFP_ACCOUNT);
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);

	return ptr;
}

void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
		       gfp_t flags)
{
	struct mem_cgroup *memcg, *old_memcg;
	void *ptr;

	memcg = bpf_map_get_memcg(map);
	old_memcg = set_active_memcg(memcg);
	ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);

	return ptr;
}

void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
				    size_t align, gfp_t flags)
{
	struct mem_cgroup *memcg, *old_memcg;
	void __percpu *ptr;

	memcg = bpf_map_get_memcg(map);
	old_memcg = set_active_memcg(memcg);
	ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);

	return ptr;
}

#else
static void bpf_map_save_memcg(struct bpf_map *map)
{
}

static void bpf_map_release_memcg(struct bpf_map *map)
{
}
#endif

static bool can_alloc_pages(void)
{
	return preempt_count() == 0 && !irqs_disabled() &&
		!IS_ENABLED(CONFIG_PREEMPT_RT);
}

static struct page *__bpf_alloc_page(int nid)
{
	if (!can_alloc_pages())
		return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);

	return alloc_pages_node(nid,
				GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT
				| __GFP_NOWARN,
				0);
}

int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
			unsigned long nr_pages, struct page **pages)
{
	unsigned long i, j;
	struct page *pg;
	int ret = 0;
#ifdef CONFIG_MEMCG
	struct mem_cgroup *memcg, *old_memcg;

	memcg = bpf_map_get_memcg(map);
	old_memcg = set_active_memcg(memcg);
#endif
	for (i = 0; i < nr_pages; i++) {
		pg = __bpf_alloc_page(nid);

		if (pg) {
			pages[i] = pg;
			continue;
		}
		for (j = 0; j < i; j++)
			free_pages_nolock(pages[j], 0);
		ret = -ENOMEM;
		break;
	}

#ifdef CONFIG_MEMCG
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);
#endif
	return ret;
}


static int btf_field_cmp(const void *a, const void *b)
{
	const struct btf_field *f1 = a, *f2 = b;

	if (f1->offset < f2->offset)
		return -1;
	else if (f1->offset > f2->offset)
		return 1;
	return 0;
}

struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,
				  u32 field_mask)
{
	struct btf_field *field;

	if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))
		return NULL;
	field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);
	if (!field || !(field->type & field_mask))
		return NULL;
	return field;
}

void btf_record_free(struct btf_record *rec)
{
	int i;

	if (IS_ERR_OR_NULL(rec))
		return;
	for (i = 0; i < rec->cnt; i++) {
		switch (rec->fields[i].type) {
		case BPF_KPTR_UNREF:
		case BPF_KPTR_REF:
		case BPF_KPTR_PERCPU:
		case BPF_UPTR:
			if (rec->fields[i].kptr.module)
				module_put(rec->fields[i].kptr.module);
			if (btf_is_kernel(rec->fields[i].kptr.btf))
				btf_put(rec->fields[i].kptr.btf);
			break;
		case BPF_LIST_HEAD:
		case BPF_LIST_NODE:
		case BPF_RB_ROOT:
		case BPF_RB_NODE:
		case BPF_SPIN_LOCK:
		case BPF_RES_SPIN_LOCK:
		case BPF_TIMER:
		case BPF_REFCOUNT:
		case BPF_WORKQUEUE:
		case BPF_TASK_WORK:
			/* Nothing to release */
			break;
		default:
			WARN_ON_ONCE(1);
			continue;
		}
	}
	kfree(rec);
}

void bpf_map_free_record(struct bpf_map *map)
{
	btf_record_free(map->record);
	map->record = NULL;
}

struct btf_record *btf_record_dup(const struct btf_record *rec)
{
	const struct btf_field *fields;
	struct btf_record *new_rec;
	int ret, size, i;

	if (IS_ERR_OR_NULL(rec))
		return NULL;
	size = struct_size(rec, fields, rec->cnt);
	new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
	if (!new_rec)
		return ERR_PTR(-ENOMEM);
	/* Do a deep copy of the btf_record */
	fields = rec->fields;
	new_rec->cnt = 0;
	for (i = 0; i < rec->cnt; i++) {
		switch (fields[i].type) {
		case BPF_KPTR_UNREF:
		case BPF_KPTR_REF:
		case BPF_KPTR_PERCPU:
		case BPF_UPTR:
			if (btf_is_kernel(fields[i].kptr.btf))
				btf_get(fields[i].kptr.btf);
			if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
				ret = -ENXIO;
				goto free;
			}
			break;
		case BPF_LIST_HEAD:
		case BPF_LIST_NODE:
		case BPF_RB_ROOT:
		case BPF_RB_NODE:
		case BPF_SPIN_LOCK:
		case BPF_RES_SPIN_LOCK:
		case BPF_TIMER:
		case BPF_REFCOUNT:
		case BPF_WORKQUEUE:
		case BPF_TASK_WORK:
			/* Nothing to acquire */
			break;
		default:
			ret = -EFAULT;
			WARN_ON_ONCE(1);
			goto free;
		}
		new_rec->cnt++;
	}
	return new_rec;
free:
	btf_record_free(new_rec);
	return ERR_PTR(ret);
}

bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)
{
	bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);
	int size;

	if (!a_has_fields && !b_has_fields)
		return true;
	if (a_has_fields != b_has_fields)
		return false;
	if (rec_a->cnt != rec_b->cnt)
		return false;
	size = struct_size(rec_a, fields, rec_a->cnt);
	/* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
	 * members are zeroed out. So memcmp is safe to do without worrying
	 * about padding/unused fields.
	 *
	 * While spin_lock, timer, and kptr have no relation to map BTF,
	 * list_head metadata is specific to map BTF, the btf and value_rec
	 * members in particular. btf is the map BTF, while value_rec points to
	 * btf_record in that map BTF.
	 *
	 * So while by default, we don't rely on the map BTF (which the records
	 * were parsed from) matching for both records, which is not backwards
	 * compatible, in case list_head is part of it, we implicitly rely on
	 * that by way of depending on memcmp succeeding for it.
	 */
	return !memcmp(rec_a, rec_b, size);
}

void bpf_obj_free_timer(const struct btf_record *rec, void *obj)
{
	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))
		return;
	bpf_timer_cancel_and_free(obj + rec->timer_off);
}

void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj)
{
	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_WORKQUEUE)))
		return;
	bpf_wq_cancel_and_free(obj + rec->wq_off);
}

void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)
{
	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TASK_WORK)))
		return;
	bpf_task_work_cancel_and_free(obj + rec->task_work_off);
}

void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
{
	const struct btf_field *fields;
	int i;

	if (IS_ERR_OR_NULL(rec))
		return;
	fields = rec->fields;
	for (i = 0; i < rec->cnt; i++) {
		struct btf_struct_meta *pointee_struct_meta;
		const struct btf_field *field = &fields[i];
		void *field_ptr = obj + field->offset;
		void *xchgd_field;

		switch (fields[i].type) {
		case BPF_SPIN_LOCK:
		case BPF_RES_SPIN_LOCK:
			break;
		case BPF_TIMER:
			bpf_timer_cancel_and_free(field_ptr);
			break;
		case BPF_WORKQUEUE:
			bpf_wq_cancel_and_free(field_ptr);
			break;
		case BPF_TASK_WORK:
			bpf_task_work_cancel_and_free(field_ptr);
			break;
		case BPF_KPTR_UNREF:
			WRITE_ONCE(*(u64 *)field_ptr, 0);
			break;
		case BPF_KPTR_REF:
		case BPF_KPTR_PERCPU:
			xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
			if (!xchgd_field)
				break;

			if (!btf_is_kernel(field->kptr.btf)) {
				pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,
									   field->kptr.btf_id);
				__bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?
								 pointee_struct_meta->record : NULL,
								 fields[i].type == BPF_KPTR_PERCPU);
			} else {
				field->kptr.dtor(xchgd_field);
			}
			break;
		case BPF_UPTR:
			/* The caller ensured that no one is using the uptr */
			unpin_uptr_kaddr(*(void **)field_ptr);
			break;
		case BPF_LIST_HEAD:
			if (WARN_ON_ONCE(rec->spin_lock_off < 0))
				continue;
			bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);
			break;
		case BPF_RB_ROOT:
			if (WARN_ON_ONCE(rec->spin_lock_off < 0))
				continue;
			bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);
			break;
		case BPF_LIST_NODE:
		case BPF_RB_NODE:
		case BPF_REFCOUNT:
			break;
		default:
			WARN_ON_ONCE(1);
			continue;
		}
	}
}

static void bpf_map_free(struct bpf_map *map)
{
	struct btf_record *rec = map->record;
	struct btf *btf = map->btf;

	/* implementation dependent freeing. Disabling migration to simplify
	 * the free of values or special fields allocated from bpf memory
	 * allocator.
	 */
	kfree(map->excl_prog_sha);
	migrate_disable();
	map->ops->map_free(map);
	migrate_enable();

	/* Delay freeing of btf_record for maps, as map_free
	 * callback usually needs access to them. It is better to do it here
	 * than require each callback to do the free itself manually.
	 *
	 * Note that the btf_record stashed in map->inner_map_meta->record was
	 * already freed using the map_free callback for map in map case which
	 * eventually calls bpf_map_free_meta, since inner_map_meta is only a
	 * template bpf_map struct used during verification.
	 */
	btf_record_free(rec);
	/* Delay freeing of btf for maps, as map_free callback may need
	 * struct_meta info which will be freed with btf_put().
	 */
	btf_put(btf);
}

/* called from workqueue */
static void bpf_map_free_deferred(struct work_struct *work)
{
	struct bpf_map *map = container_of(work, struct bpf_map, work);

	security_bpf_map_free(map);
	bpf_map_release_memcg(map);
	bpf_map_owner_free(map);
	bpf_map_free(map);
}

static void bpf_map_put_uref(struct bpf_map *map)
{
	if (atomic64_dec_and_test(&map->usercnt)) {
		if (map->ops->map_release_uref)
			map->ops->map_release_uref(map);
	}
}

static void bpf_map_free_in_work(struct bpf_map *map)
{
	INIT_WORK(&map->work, bpf_map_free_deferred);
	/* Avoid spawning kworkers, since they all might contend
	 * for the same mutex like slab_mutex.
	 */
	queue_work(system_dfl_wq, &map->work);
}

static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
{
	bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
}

static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
{
	if (rcu_trace_implies_rcu_gp())
		bpf_map_free_rcu_gp(rcu);
	else
		call_rcu(rcu, bpf_map_free_rcu_gp);
}

/* decrement map refcnt and schedule it for freeing via workqueue
 * (underlying map implementation ops->map_free() might sleep)
 */
void bpf_map_put(struct bpf_map *map)
{
	if (atomic64_dec_and_test(&map->refcnt)) {
		/* bpf_map_free_id() must be called first */
		bpf_map_free_id(map);

		WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));
		if (READ_ONCE(map->free_after_mult_rcu_gp))
			call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
		else if (READ_ONCE(map->free_after_rcu_gp))
			call_rcu(&map->rcu, bpf_map_free_rcu_gp);
		else
			bpf_map_free_in_work(map);
	}
}
EXPORT_SYMBOL_GPL(bpf_map_put);

void bpf_map_put_with_uref(struct bpf_map *map)
{
	bpf_map_put_uref(map);
	bpf_map_put(map);
}

static int bpf_map_release(struct inode *inode, struct file *filp)
{
	struct bpf_map *map = filp->private_data;

	if (map->ops->map_release)
		map->ops->map_release(map, filp);

	bpf_map_put_with_uref(map);
	return 0;
}

static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
{
	fmode_t mode = fd_file(f)->f_mode;

	/* Our file permissions may have been overridden by global
	 * map permissions facing syscall side.
	 */
	if (READ_ONCE(map->frozen))
		mode &= ~FMODE_CAN_WRITE;
	return mode;