bpf/progs/irq.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"

unsigned long global_flags;

extern void bpf_local_irq_save(unsigned long *) __weak __ksym;
extern void bpf_local_irq_restore(unsigned long *) __weak __ksym;
extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym;

struct bpf_res_spin_lock lockA __hidden SEC(".data.A");
struct bpf_res_spin_lock lockB __hidden SEC(".data.B");

SEC("?tc")
__failure __msg("arg#0 doesn't point to an irq flag on stack")
int irq_save_bad_arg(struct __sk_buff *ctx)
{
	bpf_local_irq_save(&global_flags);
	return 0;
}

SEC("?tc")
__failure __msg("arg#0 doesn't point to an irq flag on stack")
int irq_restore_bad_arg(struct __sk_buff *ctx)
{
	bpf_local_irq_restore(&global_flags);
	return 0;
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_2(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_save(&flags2);
	return 0;
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_3(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_save(&flags2);
	bpf_local_irq_save(&flags3);
	return 0;
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_3_minus_2(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_save(&flags2);
	bpf_local_irq_save(&flags3);
	bpf_local_irq_restore(&flags3);
	bpf_local_irq_restore(&flags2);
	return 0;
}

static __noinline void local_irq_save(unsigned long *flags)
{
	bpf_local_irq_save(flags);
}

static __noinline void local_irq_restore(unsigned long *flags)
{
	bpf_local_irq_restore(flags);
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_1_subprog(struct __sk_buff *ctx)
{
	unsigned long flags;

	local_irq_save(&flags);
	return 0;
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_2_subprog(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;

	local_irq_save(&flags1);
	local_irq_save(&flags2);
	return 0;
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_3_subprog(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	local_irq_save(&flags1);
	local_irq_save(&flags2);
	local_irq_save(&flags3);
	return 0;
}

SEC("?tc")
__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	local_irq_save(&flags1);
	local_irq_save(&flags2);
	local_irq_save(&flags3);
	local_irq_restore(&flags3);
	local_irq_restore(&flags2);
	return 0;
}

SEC("?tc")
__success
int irq_balance(struct __sk_buff *ctx)
{
	unsigned long flags;

	local_irq_save(&flags);
	local_irq_restore(&flags);
	return 0;
}

SEC("?tc")
__success
int irq_balance_n(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	local_irq_save(&flags1);
	local_irq_save(&flags2);
	local_irq_save(&flags3);
	local_irq_restore(&flags3);
	local_irq_restore(&flags2);
	local_irq_restore(&flags1);
	return 0;
}

static __noinline void local_irq_balance(void)
{
	unsigned long flags;

	local_irq_save(&flags);
	local_irq_restore(&flags);
}

static __noinline void local_irq_balance_n(void)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	local_irq_save(&flags1);
	local_irq_save(&flags2);
	local_irq_save(&flags3);
	local_irq_restore(&flags3);
	local_irq_restore(&flags2);
	local_irq_restore(&flags1);
}

SEC("?tc")
__success
int irq_balance_subprog(struct __sk_buff *ctx)
{
	local_irq_balance();
	return 0;
}

SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
__failure __msg("sleepable helper bpf_copy_from_user#")
int irq_sleepable_helper(void *ctx)
{
	unsigned long flags;
	u32 data;

	local_irq_save(&flags);
	bpf_copy_from_user(&data, sizeof(data), NULL);
	local_irq_restore(&flags);
	return 0;
}

SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
__failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region")
int irq_sleepable_kfunc(void *ctx)
{
	unsigned long flags;
	u32 data;

	local_irq_save(&flags);
	bpf_copy_from_user_str(&data, sizeof(data), NULL, 0);
	local_irq_restore(&flags);
	return 0;
}

int __noinline global_local_irq_balance(void)
{
	local_irq_balance_n();
	return 0;
}

SEC("?tc")
__success
int irq_global_subprog(struct __sk_buff *ctx)
{
	unsigned long flags;

	bpf_local_irq_save(&flags);
	global_local_irq_balance();
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?tc")
__failure __msg("cannot restore irq state out of order")
int irq_restore_ooo(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_save(&flags2);
	bpf_local_irq_restore(&flags1);
	bpf_local_irq_restore(&flags2);
	return 0;
}

SEC("?tc")
__failure __msg("cannot restore irq state out of order")
int irq_restore_ooo_3(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_save(&flags2);
	bpf_local_irq_restore(&flags2);
	bpf_local_irq_save(&flags3);
	bpf_local_irq_restore(&flags1);
	bpf_local_irq_restore(&flags3);
	return 0;
}

static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2,
					unsigned long *flags3)
{
	local_irq_save(flags1);
	local_irq_save(flags2);
	local_irq_save(flags3);
}

SEC("?tc")
__success
int irq_restore_3_subprog(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	local_irq_save_3(&flags1, &flags2, &flags3);
	bpf_local_irq_restore(&flags3);
	bpf_local_irq_restore(&flags2);
	bpf_local_irq_restore(&flags1);
	return 0;
}

SEC("?tc")
__failure __msg("cannot restore irq state out of order")
int irq_restore_4_subprog(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;
	unsigned long flags4;

	local_irq_save_3(&flags1, &flags2, &flags3);
	bpf_local_irq_restore(&flags3);
	bpf_local_irq_save(&flags4);
	bpf_local_irq_restore(&flags4);
	bpf_local_irq_restore(&flags1);
	return 0;
}

SEC("?tc")
__failure __msg("cannot restore irq state out of order")
int irq_restore_ooo_3_subprog(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags2;
	unsigned long flags3;

	local_irq_save_3(&flags1, &flags2, &flags3);
	bpf_local_irq_restore(&flags3);
	bpf_local_irq_restore(&flags2);
	bpf_local_irq_save(&flags3);
	bpf_local_irq_restore(&flags1);
	return 0;
}

SEC("?tc")
__failure __msg("expected an initialized")
int irq_restore_invalid(struct __sk_buff *ctx)
{
	unsigned long flags1;
	unsigned long flags = 0xfaceb00c;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?tc")
__failure __msg("expected uninitialized")
int irq_save_invalid(struct __sk_buff *ctx)
{
	unsigned long flags1;

	bpf_local_irq_save(&flags1);
	bpf_local_irq_save(&flags1);
	return 0;
}

SEC("?tc")
__failure __msg("expected an initialized")
int irq_restore_iter(struct __sk_buff *ctx)
{
	struct bpf_iter_num it;

	bpf_iter_num_new(&it, 0, 42);
	bpf_local_irq_restore((unsigned long *)&it);
	return 0;
}

SEC("?tc")
__failure __msg("Unreleased reference id=1")
int irq_save_iter(struct __sk_buff *ctx)
{
	struct bpf_iter_num it;

	/* Ensure same sized slot has st->ref_obj_id set, so we reject based on
	 * slot_type != STACK_IRQ_FLAG...
	 */
	_Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size");

	bpf_iter_num_new(&it, 0, 42);
	bpf_local_irq_save((unsigned long *)&it);
	bpf_local_irq_restore((unsigned long *)&it);
	return 0;
}

SEC("?tc")
__failure __msg("expected an initialized")
int irq_flag_overwrite(struct __sk_buff *ctx)
{
	unsigned long flags;

	bpf_local_irq_save(&flags);
	flags = 0xdeadbeef;
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?tc")
__failure __msg("expected an initialized")
int irq_flag_overwrite_partial(struct __sk_buff *ctx)
{
	unsigned long flags;

	bpf_local_irq_save(&flags);
	*(((char *)&flags) + 1) = 0xff;
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?tc")
__failure __msg("cannot restore irq state out of order")
int irq_ooo_refs_array(struct __sk_buff *ctx)
{
	unsigned long flags[4];
	struct { int i; } *p;

	/* refs=1 */
	bpf_local_irq_save(&flags[0]);

	/* refs=1,2 */
	p = bpf_obj_new(typeof(*p));
	if (!p) {
		bpf_local_irq_restore(&flags[0]);
		return 0;
	}

	/* refs=1,2,3 */
	bpf_local_irq_save(&flags[1]);

	/* refs=1,2,3,4 */
	bpf_local_irq_save(&flags[2]);

	/* Now when we remove ref=2, the verifier must not break the ordering in
	 * the refs array between 1,3,4. With an older implementation, the
	 * verifier would swap the last element with the removed element, but to
	 * maintain the stack property we need to use memmove.
	 */
	bpf_obj_drop(p);

	/* Save and restore to reset active_irq_id to 3, as the ordering is now
	 * refs=1,4,3. When restoring the linear scan will find prev_id in order
	 * as 3 instead of 4.
	 */
	bpf_local_irq_save(&flags[3]);
	bpf_local_irq_restore(&flags[3]);

	/* With the incorrect implementation, we can release flags[1], flags[2],
	 * and flags[0], i.e. in the wrong order.
	 */
	bpf_local_irq_restore(&flags[1]);
	bpf_local_irq_restore(&flags[2]);
	bpf_local_irq_restore(&flags[0]);
	return 0;
}

int __noinline
global_subprog(int i)
{
	if (i)
		bpf_printk("%p", &i);
	return i;
}

int __noinline
global_sleepable_helper_subprog(int i)
{
	if (i)
		bpf_copy_from_user(&i, sizeof(i), NULL);
	return i;
}

int __noinline
global_sleepable_kfunc_subprog(int i)
{
	if (i)
		bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
	global_subprog(i);
	return i;
}

int __noinline
global_subprog_calling_sleepable_global(int i)
{
	if (!i)
		global_sleepable_kfunc_subprog(i);
	return i;
}

SEC("?syscall")
__success
int irq_non_sleepable_global_subprog(void *ctx)
{
	unsigned long flags;

	bpf_local_irq_save(&flags);
	global_subprog(0);
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?syscall")
__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
int irq_sleepable_helper_global_subprog(void *ctx)
{
	unsigned long flags;

	bpf_local_irq_save(&flags);
	global_sleepable_helper_subprog(0);
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?syscall")
__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
int irq_sleepable_global_subprog_indirect(void *ctx)
{
	unsigned long flags;

	bpf_local_irq_save(&flags);
	global_subprog_calling_sleepable_global(0);
	bpf_local_irq_restore(&flags);
	return 0;
}

SEC("?tc")
__failure __msg("cannot restore irq state out of order")
int irq_ooo_lock_cond_inv(struct __sk_buff *ctx)
{
	unsigned long flags1, flags2;

	if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
		return 0;
	if (bpf_res_spin_lock_irqsave(&lockB, &flags2)) {
		bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
		return 0;
	}

	bpf_res_spin_unlock_irqrestore(&lockB, &flags1);
	bpf_res_spin_unlock_irqrestore(&lockA, &flags2);
	return 0;
}

SEC("?tc")
__failure __msg("function calls are not allowed")
int irq_wrong_kfunc_class_1(struct __sk_buff *ctx)
{
	unsigned long flags1;

	if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
		return 0;
	/* For now, bpf_local_irq_restore is not allowed in critical section,
	 * but this test ensures error will be caught with kfunc_class when it's
	 * opened up. Tested by temporarily permitting this kfunc in critical
	 * section.
	 */
	bpf_local_irq_restore(&flags1);
	bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
	return 0;
}

SEC("?tc")
__failure __msg("function calls are not allowed")
int irq_wrong_kfunc_class_2(struct __sk_buff *ctx)
{
	unsigned long flags1, flags2;

	bpf_local_irq_save(&flags1);
	if (bpf_res_spin_lock_irqsave(&lockA, &flags2))
		return 0;
	bpf_local_irq_restore(&flags2);
	bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
	return 0;
}

char _license[] SEC("license") = "GPL";