diff --git a/src/bpfilter/CMakeLists.txt b/src/bpfilter/CMakeLists.txt index d630c1368..2024ebe1d 100644 --- a/src/bpfilter/CMakeLists.txt +++ b/src/bpfilter/CMakeLists.txt @@ -18,14 +18,10 @@ add_executable(bpfilter ${CMAKE_CURRENT_SOURCE_DIR}/cgen/fixup.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/fixup.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/handle.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/handle.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/jmp.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/jmp.c - ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/ip4.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/ip4.c - ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/ip6.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/ip6.c - ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/tcp.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/tcp.c - ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/udp.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/udp.c + ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/cmp.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/cmp.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/meta.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/meta.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/packet.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/packet.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/set.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/set.c - ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/icmp.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/icmp.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/nf.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/nf.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/printer.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/printer.c ${CMAKE_CURRENT_SOURCE_DIR}/cgen/program.h ${CMAKE_CURRENT_SOURCE_DIR}/cgen/program.c diff --git a/src/bpfilter/cgen/cgroup_skb.c b/src/bpfilter/cgen/cgroup_skb.c index 7d3330ea2..f35285c1d 100644 --- a/src/bpfilter/cgen/cgroup_skb.c +++ b/src/bpfilter/cgen/cgroup_skb.c @@ -19,7 +19,7 @@ #include #include "cgen/cgen.h" -#include "cgen/matcher/meta.h" +#include "cgen/matcher/cmp.h" #include "cgen/matcher/packet.h" #include "cgen/program.h" #include "cgen/stub.h" @@ -130,12 +130,20 @@ static int _bf_cgroup_skb_gen_inline_matcher(struct bf_program *program, EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, offsetof(struct __sk_buff, mark))); - return bf_matcher_generate_meta_mark_cmp(program, matcher); + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 4, BPF_REG_1); case BF_MATCHER_META_FLOW_HASH: EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg))); + EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_hash_recalc)); - return bf_matcher_generate_meta_flow_hash_cmp(program, matcher); + if (bf_matcher_get_op(matcher) == BF_MATCHER_RANGE) { + uint32_t *hash = (uint32_t *)bf_matcher_payload(matcher); + return bf_cmp_range(program, hash[0], hash[1], BPF_REG_0); + } + + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 4, BPF_REG_0); default: return bf_matcher_generate_packet(program, matcher); } diff --git a/src/bpfilter/cgen/matcher/cmp.c b/src/bpfilter/cgen/matcher/cmp.c new file mode 100644 index 000000000..1737590ac --- /dev/null +++ b/src/bpfilter/cgen/matcher/cmp.c @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + */ + +#include "cgen/matcher/cmp.h" + +#include +#include + +#include +#include + +#include +#include + +#include "cgen/jmp.h" +#include "cgen/program.h" + +#define _BF_MASK_LAST_BYTE 15 + +static inline uint64_t _bf_read_u64(const void *ptr) +{ + uint64_t val; + + memcpy(&val, ptr, sizeof(val)); + + return val; +} + +/** + * @brief Emit a 4-instruction sequence to build a 64-bit immediate from 8 bytes. + * + * Produces: + * @code + * MOV32_IMM(dst, high32) -> LSH(dst, 32) -> MOV32_IMM(scratch, low32) -> OR(dst, scratch) + * @endcode + * + * @param program Program to emit into. Can't be NULL. + * @param dst_reg Destination register for the 64-bit value. + * @param scratch_reg Scratch register (clobbered). + * @param data 64-bit value to load. + */ +static int _bf_cmp_build_imm64(struct bf_program *program, int dst_reg, + int scratch_reg, uint64_t data) +{ + EMIT(program, BPF_MOV32_IMM(dst_reg, (uint32_t)(data >> 32))); + EMIT(program, BPF_ALU64_IMM(BPF_LSH, dst_reg, 32)); + EMIT(program, BPF_MOV32_IMM(scratch_reg, (uint32_t)data)); + EMIT(program, BPF_ALU64_REG(BPF_OR, dst_reg, scratch_reg)); + + return 0; +} + +/** + * @brief Compute a network prefix mask. + * + * @param prefixlen Prefix length in bits. + * @param mask Output buffer. Can't be NULL. + * @param mask_len Size of mask buffer in bytes (4 or 16). + */ +static void _bf_prefix_to_mask(unsigned int prefixlen, uint8_t *mask, + size_t mask_len) +{ + assert(mask); + + memset(mask, 0x00, mask_len); + memset(mask, 0xff, prefixlen / 8); + if (prefixlen % 8) + mask[prefixlen / 8] = (0xff << (8 - prefixlen % 8)) & 0xff; +} + +int bf_cmp_value(struct bf_program *program, enum bf_matcher_op op, + const void *ref, unsigned int size, int reg) +{ + assert(program); + assert(ref); + + if (op != BF_MATCHER_EQ && op != BF_MATCHER_NE) + return bf_err_r(-EINVAL, "unsupported operator %d", op); + + switch (size) { + case 1: + case 2: { + /* Small values: compare directly via JMP_IMM. + * For size 1, ref is uint8_t; for size 2, ref is uint16_t. + * Both fit in a signed 32-bit immediate. */ + uint32_t val = + (size == 1) ? *(const uint8_t *)ref : *(const uint16_t *)ref; + + EMIT_FIXUP_JMP_NEXT_RULE( + program, + BPF_JMP_IMM(op == BF_MATCHER_EQ ? BPF_JNE : BPF_JEQ, reg, val, 0)); + break; + } + case 4: { + /* 32-bit values: may exceed signed 32-bit immediate range, so + * use MOV32_IMM into R2 + JMP_REG. */ + uint32_t val = *(const uint32_t *)ref; + + EMIT(program, BPF_MOV32_IMM(BPF_REG_2, val)); + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_REG(op == BF_MATCHER_EQ ? BPF_JNE : BPF_JEQ, reg, + BPF_REG_2, 0)); + break; + } + case 8: { + /* 64-bit values: build immediate in R2 via `_bf_cmp_build_imm64`, + * then compare with `JMP_REG`. */ + int r; + + r = _bf_cmp_build_imm64(program, BPF_REG_2, BPF_REG_3, + _bf_read_u64(ref)); + if (r) + return r; + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_REG(op == BF_MATCHER_EQ ? BPF_JNE : BPF_JEQ, reg, + BPF_REG_2, 0)); + break; + } + case 16: { + /* 128-bit values: reg holds low 64 bits, reg+1 holds high 64 bits. + * Compare each half against the reference. */ + const uint8_t *addr = ref; + int r; + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(addr)); + if (r) + return r; + + if (op == BF_MATCHER_EQ) { + EMIT_FIXUP_JMP_NEXT_RULE(program, + BPF_JMP_REG(BPF_JNE, reg, BPF_REG_3, 0)); + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(addr + 8)); + if (r) + return r; + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_REG(BPF_JNE, reg + 1, BPF_REG_3, 0)); + } else { + /* NE: the address must differ in at least one half. + * If the first half differs, the matcher matched — jump + * past the second half check and the unconditional + * jump-to-next-rule. If the first half matches, check the + * second half: if it also matches, the full address is + * equal, so the NE matcher fails — jump to next rule. */ + _clean_bf_jmpctx_ struct bf_jmpctx j0 = bf_jmpctx_default(); + _clean_bf_jmpctx_ struct bf_jmpctx j1 = bf_jmpctx_default(); + + j0 = + bf_jmpctx_get(program, BPF_JMP_REG(BPF_JNE, reg, BPF_REG_3, 0)); + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(addr + 8)); + if (r) + return r; + j1 = bf_jmpctx_get(program, + BPF_JMP_REG(BPF_JNE, reg + 1, BPF_REG_3, 0)); + + EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP_A(0)); + } + break; + } + default: + return bf_err_r(-EINVAL, "unsupported comparison size %u", size); + } + + return 0; +} + +int bf_cmp_masked_value(struct bf_program *program, enum bf_matcher_op op, + const void *ref, unsigned int prefixlen, + unsigned int size, int reg) +{ + assert(program); + assert(ref); + + if (op != BF_MATCHER_EQ && op != BF_MATCHER_NE) + return bf_err_r(-EINVAL, "unsupported operator %d", op); + + switch (size) { + case 4: { + uint32_t mask; + const uint32_t *addr = ref; + + _bf_prefix_to_mask(prefixlen, (uint8_t *)&mask, 4); + + EMIT(program, BPF_MOV32_IMM(BPF_REG_2, *addr)); + + if (mask != ~0U) { + EMIT(program, BPF_MOV32_IMM(BPF_REG_3, mask)); + EMIT(program, BPF_ALU32_REG(BPF_AND, reg, BPF_REG_3)); + EMIT(program, BPF_ALU32_REG(BPF_AND, BPF_REG_2, BPF_REG_3)); + } + + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_REG(op == BF_MATCHER_EQ ? BPF_JNE : BPF_JEQ, reg, + BPF_REG_2, 0)); + break; + } + case 16: { + uint8_t mask[16]; + uint8_t masked_lo[8], masked_hi[8]; + const uint8_t *addr = ref; + int r; + + _bf_prefix_to_mask(prefixlen, mask, 16); + + // Apply mask to loaded reg/reg+1 if not a full /128 + if (mask[_BF_MASK_LAST_BYTE] != (uint8_t)~0) { + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(mask)); + if (r) + return r; + EMIT(program, BPF_ALU64_REG(BPF_AND, reg, BPF_REG_3)); + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(mask + 8)); + if (r) + return r; + EMIT(program, BPF_ALU64_REG(BPF_AND, reg + 1, BPF_REG_3)); + } + + for (int i = 0; i < 8; i++) + masked_lo[i] = addr[i] & mask[i]; + for (int i = 0; i < 8; i++) + masked_hi[i] = addr[i + 8] & mask[i + 8]; + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(masked_lo)); + if (r) + return r; + + if (op == BF_MATCHER_EQ) { + EMIT_FIXUP_JMP_NEXT_RULE(program, + BPF_JMP_REG(BPF_JNE, reg, BPF_REG_3, 0)); + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(masked_hi)); + if (r) + return r; + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_REG(BPF_JNE, reg + 1, BPF_REG_3, 0)); + } else { + _clean_bf_jmpctx_ struct bf_jmpctx j0 = bf_jmpctx_default(); + _clean_bf_jmpctx_ struct bf_jmpctx j1 = bf_jmpctx_default(); + + j0 = + bf_jmpctx_get(program, BPF_JMP_REG(BPF_JNE, reg, BPF_REG_3, 0)); + + r = _bf_cmp_build_imm64(program, BPF_REG_3, BPF_REG_4, + _bf_read_u64(masked_hi)); + if (r) + return r; + j1 = bf_jmpctx_get(program, + BPF_JMP_REG(BPF_JNE, reg + 1, BPF_REG_3, 0)); + + EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP_A(0)); + } + break; + } + default: + return bf_err_r(-EINVAL, "unsupported masked comparison size %u", size); + } + + return 0; +} + +int bf_cmp_range(struct bf_program *program, uint32_t min, uint32_t max, + int reg) +{ + assert(program); + + EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP32_IMM(BPF_JLT, reg, min, 0)); + EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP32_IMM(BPF_JGT, reg, max, 0)); + + return 0; +} + +int bf_cmp_bitfield(struct bf_program *program, enum bf_matcher_op op, + uint32_t flags, int reg) +{ + assert(program); + + if (op != BF_MATCHER_ANY && op != BF_MATCHER_ALL) + return bf_err_r(-EINVAL, "unsupported operator %d", op); + + EMIT(program, BPF_ALU32_IMM(BPF_AND, reg, flags)); + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP32_IMM(op == BF_MATCHER_ANY ? BPF_JEQ : BPF_JNE, reg, + op == BF_MATCHER_ANY ? 0 : flags, 0)); + + return 0; +} diff --git a/src/bpfilter/cgen/matcher/cmp.h b/src/bpfilter/cgen/matcher/cmp.h new file mode 100644 index 000000000..0930d0a92 --- /dev/null +++ b/src/bpfilter/cgen/matcher/cmp.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + */ + +#pragma once + +#include + +#include + +struct bf_program; + +/** + * @brief Compare the value in `reg` against a reference value. + * + * For size 16, the value spans `reg` (low 64 bits) and `reg + 1` + * (high 64 bits). Clobbers `BPF_REG_2` for size 4/8 and + * `BPF_REG_3`/`BPF_REG_4` for size 16. + * + * @param program Program to emit into. Can't be NULL. + * @param op `BF_MATCHER_EQ` or `BF_MATCHER_NE`. + * @param ref Pointer to reference value. Can't be NULL. + * @param size Comparison width in bytes: 1, 2, 4, 8, or 16. + * @param reg BPF register holding the value to compare. + * @return 0 on success, negative errno on error. + */ +int bf_cmp_value(struct bf_program *program, enum bf_matcher_op op, + const void *ref, unsigned int size, int reg); + +/** + * @brief Mask the value in `reg` by `prefixlen`, then compare. + * + * For size 16, the value spans `reg` and `reg + 1`. Modifies `reg` + * (and `reg + 1`) in-place. Clobbers `BPF_REG_2`/`BPF_REG_3` for + * size 4 and `BPF_REG_3`/`BPF_REG_4` for size 16. + * + * @param program Program to emit into. Can't be NULL. + * @param op `BF_MATCHER_EQ` or `BF_MATCHER_NE`. + * @param ref Pointer to unmasked reference value. Can't be NULL. + * @param prefixlen Prefix length in bits (1-32 for size 4, 1-128 for size 16). + * @param size Comparison width in bytes: 4 or 16. + * @param reg BPF register holding the value to compare. + * @return 0 on success, negative errno on error. + */ +int bf_cmp_masked_value(struct bf_program *program, enum bf_matcher_op op, + const void *ref, unsigned int prefixlen, + unsigned int size, int reg); + +/** + * @brief Check `min <= reg <= max`, for values up to 32 bits. + * + * All values must be in host byte order; the caller is responsible for + * any conversion (e.g., `BSWAP` for network-order port values). + * + * @param program Program to emit into. Can't be NULL. + * @param min Minimum value. + * @param max Maximum value. + * @param reg BPF register holding the value to compare. + * @return 0 on success, negative errno on error. + */ +int bf_cmp_range(struct bf_program *program, uint32_t min, uint32_t max, + int reg); + +/** + * @brief Check `reg` against a bitmask, for values up to 32 bits. + * + * ANY: `(reg & flags) != 0` + * ALL: `(reg & flags) == flags` + * + * @param program Program to emit into. Can't be NULL. + * @param op `BF_MATCHER_ANY` or `BF_MATCHER_ALL`. + * @param flags Bitmask to check against. + * @param reg BPF register holding the value to check. + * @return 0 on success, negative errno on error. + */ +int bf_cmp_bitfield(struct bf_program *program, enum bf_matcher_op op, + uint32_t flags, int reg); diff --git a/src/bpfilter/cgen/matcher/icmp.c b/src/bpfilter/cgen/matcher/icmp.c deleted file mode 100644 index e791838ef..000000000 --- a/src/bpfilter/cgen/matcher/icmp.c +++ /dev/null @@ -1,94 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#include -#include -#include -#include -#include - -#include - -#include - -#include "cgen/program.h" - -static int _bf_matcher_generate_icmp_fields(struct bf_program *program, - const struct bf_matcher *matcher, - const size_t offset) -{ - const uint8_t value = *(uint8_t *)bf_matcher_payload(matcher); - - EMIT(program, BPF_LDX_MEM(BPF_B, BPF_REG_1, BPF_REG_6, offset)); - - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, value, 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, value, 0)); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher operator '%s' (%d)", - bf_matcher_op_to_str(bf_matcher_get_op(matcher)), - bf_matcher_get_op(matcher)); - } - - return 0; -} - -static int _bf_matcher_generate_icmp6_fields(struct bf_program *program, - const struct bf_matcher *matcher) -{ - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_ICMPV6_TYPE ? - offsetof(struct icmp6hdr, icmp6_type) : - offsetof(struct icmp6hdr, icmp6_code); - - EMIT_FIXUP_JMP_NEXT_RULE( - program, BPF_JMP_IMM(BPF_JNE, BPF_REG_8, IPPROTO_ICMPV6, 0)); - EMIT(program, - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, BF_PROG_CTX_OFF(l4_hdr))); - - return _bf_matcher_generate_icmp_fields(program, matcher, offset); -} - -static int _bf_matcher_generate_icmp4_fields(struct bf_program *program, - const struct bf_matcher *matcher) -{ - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_ICMP_TYPE ? - offsetof(struct icmphdr, type) : - offsetof(struct icmphdr, code); - - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_8, IPPROTO_ICMP, 0)); - EMIT(program, - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, BF_PROG_CTX_OFF(l4_hdr))); - - return _bf_matcher_generate_icmp_fields(program, matcher, offset); -} - -int bf_matcher_generate_icmp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - int r; - - switch (bf_matcher_get_type(matcher)) { - case BF_MATCHER_ICMP_TYPE: - case BF_MATCHER_ICMP_CODE: - r = _bf_matcher_generate_icmp4_fields(program, matcher); - break; - case BF_MATCHER_ICMPV6_TYPE: - case BF_MATCHER_ICMPV6_CODE: - r = _bf_matcher_generate_icmp6_fields(program, matcher); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher type %d", - bf_matcher_get_type(matcher)); - }; - - return r; -} diff --git a/src/bpfilter/cgen/matcher/icmp.h b/src/bpfilter/cgen/matcher/icmp.h deleted file mode 100644 index 0475306d4..000000000 --- a/src/bpfilter/cgen/matcher/icmp.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#pragma once - -struct bf_matcher; -struct bf_program; - -int bf_matcher_generate_icmp(struct bf_program *program, - const struct bf_matcher *matcher); diff --git a/src/bpfilter/cgen/matcher/ip4.c b/src/bpfilter/cgen/matcher/ip4.c deleted file mode 100644 index 9250c5630..000000000 --- a/src/bpfilter/cgen/matcher/ip4.c +++ /dev/null @@ -1,172 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#include "cgen/matcher/ip4.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "cgen/program.h" -#include "filter.h" - -static int _bf_matcher_generate_ip4_addr(struct bf_program *program, - const struct bf_matcher *matcher) -{ - assert(program); - assert(matcher); - - uint32_t *addr = (uint32_t *)bf_matcher_payload(matcher); - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_IP4_SADDR ? - offsetof(struct iphdr, saddr) : - offsetof(struct iphdr, daddr); - - EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, offset)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_2, *addr)); - - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_REG(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : - BPF_JEQ, - BPF_REG_1, BPF_REG_2, 0)); - - return 0; -} - -static int _bf_matcher_generate_ip4_proto(struct bf_program *program, - const struct bf_matcher *matcher) -{ - assert(program); - assert(matcher); - - uint8_t proto = *(uint8_t *)bf_matcher_payload(matcher); - - EMIT(program, BPF_LDX_MEM(BPF_B, BPF_REG_1, BPF_REG_6, - offsetof(struct iphdr, protocol))); - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_IMM(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : - BPF_JEQ, - BPF_REG_1, proto, 0)); - - return 0; -} - -static int _bf_matcher_generate_ip4_dscp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint8_t dscp; - - assert(program); - assert(matcher); - - dscp = *(uint8_t *)bf_matcher_payload(matcher); - - EMIT(program, - BPF_LDX_MEM(BPF_B, BPF_REG_1, BPF_REG_6, offsetof(struct iphdr, tos))); - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_IMM(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : - BPF_JEQ, - BPF_REG_1, dscp, 0)); - - return 0; -} - -static void _bf_ip4_prefix_to_mask(uint32_t prefixlen, uint8_t *mask) -{ - assert(mask); - - memset(mask, 0x00, 4); - - memset(mask, 0xff, prefixlen / 8); - if (prefixlen % 8) - mask[prefixlen / 8] = 0xff << (8 - prefixlen % 8) & 0xff; -} - -static int _bf_matcher_generate_ip4_net(struct bf_program *program, - const struct bf_matcher *matcher) -{ - assert(program); - assert(matcher); - - uint32_t mask; - struct bf_ip4_lpm_key *addr = - (struct bf_ip4_lpm_key *)bf_matcher_payload(matcher); - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_IP4_SNET ? - offsetof(struct iphdr, saddr) : - offsetof(struct iphdr, daddr); - - _bf_ip4_prefix_to_mask(addr->prefixlen, (void *)&mask); - - EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, offset)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_2, addr->data)); - - if (mask != ~0U) { - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, mask)); - EMIT(program, BPF_ALU32_REG(BPF_AND, BPF_REG_1, BPF_REG_3)); - EMIT(program, BPF_ALU32_REG(BPF_AND, BPF_REG_2, BPF_REG_3)); - } - - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_REG(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : - BPF_JEQ, - BPF_REG_1, BPF_REG_2, 0)); - - return 0; -} - -int bf_matcher_generate_ip4(struct bf_program *program, - const struct bf_matcher *matcher) -{ - assert(program); - assert(matcher); - - int r; - - EMIT_FIXUP_JMP_NEXT_RULE( - program, BPF_JMP_IMM(BPF_JNE, BPF_REG_7, htobe16(ETH_P_IP), 0)); - - EMIT(program, - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, BF_PROG_CTX_OFF(l3_hdr))); - - switch (bf_matcher_get_type(matcher)) { - case BF_MATCHER_IP4_SADDR: - case BF_MATCHER_IP4_DADDR: - r = _bf_matcher_generate_ip4_addr(program, matcher); - break; - case BF_MATCHER_IP4_PROTO: - r = _bf_matcher_generate_ip4_proto(program, matcher); - break; - case BF_MATCHER_IP4_DSCP: - r = _bf_matcher_generate_ip4_dscp(program, matcher); - break; - case BF_MATCHER_IP4_SNET: - case BF_MATCHER_IP4_DNET: - r = _bf_matcher_generate_ip4_net(program, matcher); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher type %d", - bf_matcher_get_type(matcher)); - }; - - if (r) - return r; - - return 0; -} diff --git a/src/bpfilter/cgen/matcher/ip4.h b/src/bpfilter/cgen/matcher/ip4.h deleted file mode 100644 index 927e83c6e..000000000 --- a/src/bpfilter/cgen/matcher/ip4.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#pragma once - -struct bf_matcher; -struct bf_program; - -/** - * Generate the bytecode for the BF_MATCHER_IP4_* matcher types. - * - * @param program Program to generate the bytecode into. Can't be NULL. - * @param matcher Matcher to generate the bytecode for. Can't be NULL. - * @return 0 on success, negative errno value on failure. - */ -int bf_matcher_generate_ip4(struct bf_program *program, - const struct bf_matcher *matcher); diff --git a/src/bpfilter/cgen/matcher/ip6.c b/src/bpfilter/cgen/matcher/ip6.c deleted file mode 100644 index a6668a22d..000000000 --- a/src/bpfilter/cgen/matcher/ip6.c +++ /dev/null @@ -1,356 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#include "cgen/matcher/ip6.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include "cgen/jmp.h" -#include "cgen/program.h" -#include "filter.h" - -#define _bf_make32(a, b, c, d) \ - (((uint32_t)(a) << 24) | ((uint32_t)(b) << 16) | ((uint32_t)(c) << 8) | \ - (uint32_t)(d)) -#define _BF_MASK_LAST_BYTE 15 -#define BF_IPV6_EH_HOPOPTS(x) ((x) << 0) -#define BF_IPV6_EH_ROUTING(x) ((x) << 1) -#define BF_IPV6_EH_FRAGMENT(x) ((x) << 2) -#define BF_IPV6_EH_AH(x) ((x) << 3) -#define BF_IPV6_EH_DSTOPTS(x) ((x) << 4) -#define BF_IPV6_EH_MH(x) ((x) << 5) - -static int _bf_matcher_generate_ip6_addr(struct bf_program *program, - const struct bf_matcher *matcher) -{ - struct bf_jmpctx j0, j1; - uint8_t *addr = (uint8_t *)bf_matcher_payload(matcher); - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_IP6_SADDR ? - offsetof(struct ipv6hdr, saddr) : - offsetof(struct ipv6hdr, daddr); - - EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, offset)); - EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_6, offset + 8)); - - if (bf_matcher_get_op(matcher) == BF_MATCHER_EQ) { - /* If we want to match an IP, both addr[0] and addr[1] - * must match the packet, otherwise we jump to the next rule. */ - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr[7], addr[6], - addr[5], addr[4]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr[3], addr[2], - addr[1], addr[0]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_3, 0)); - - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr[15], addr[14], - addr[13], addr[12]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr[11], addr[10], - addr[9], addr[8]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, 0)); - } else { - /* If we want to *not* match an IP, none of addr[0] and - * addr[1] should match the packet, otherwise we jump to the - * next rule. */ - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr[7], addr[6], - addr[5], addr[4]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr[3], addr[2], - addr[1], addr[0]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - - /* Branching: - * - addr[0] matches the address' 64 MSB: continue to compare - * the address' 64 LSB. - * - addr[0] doesn't matches the address' 64 MSB: jump to the - * end of the matcher to continue the processing of the current rule. - * This matcher matched. */ - j0 = bf_jmpctx_get(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_3, 0)); - - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr[15], addr[14], - addr[13], addr[12]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr[11], addr[10], - addr[9], addr[8]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - - /* Branching: - * - addr[1] matches the address' 64 LSB: addr matches the - * packet's address, meaning the matcher doesn't match. Jump to the - * next rule. - * - addr[1] doesn't matches the address' 64 LSB: the matcher - * matched: addr is not equal to the packet's address. Continue - * processing with the next matcher. */ - j1 = bf_jmpctx_get(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, 0)); - - EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP_A(0)); - - // j0 and j1 should jump here if they can't match the packet's IP. - bf_jmpctx_cleanup(&j0); - bf_jmpctx_cleanup(&j1); - } - - return 0; -} - -static void _bf_ip6_prefix_to_mask(uint32_t prefixlen, uint8_t *mask) -{ - assert(mask); - - memset(mask, 0x00, 16); - - memset(mask, 0xff, prefixlen / 8); - if (prefixlen % 8) - mask[prefixlen / 8] = 0xff << (8 - prefixlen % 8) & 0xff; -} - -static int _bf_matcher_generate_ip6_net(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint8_t mask[16]; - struct bf_jmpctx j0, j1; - const struct bf_ip6_lpm_key *addr = bf_matcher_payload(matcher); - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_IP6_SNET ? - offsetof(struct ipv6hdr, saddr) : - offsetof(struct ipv6hdr, daddr); - - EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, offset)); - EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_6, offset + 8)); - - _bf_ip6_prefix_to_mask(addr->prefixlen, mask); - - if (mask[_BF_MASK_LAST_BYTE] != (uint8_t)~0) { - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, _bf_make32(mask[7], mask[6], - mask[5], mask[4]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_4, _bf_make32(mask[3], mask[2], - mask[1], mask[0]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - EMIT(program, BPF_ALU64_REG(BPF_AND, BPF_REG_1, BPF_REG_3)); - - EMIT(program, BPF_MOV32_IMM(BPF_REG_3, _bf_make32(mask[15], mask[14], - mask[13], mask[12]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, BPF_MOV32_IMM(BPF_REG_4, _bf_make32(mask[11], mask[10], - mask[9], mask[8]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - EMIT(program, BPF_ALU64_REG(BPF_AND, BPF_REG_2, BPF_REG_3)); - } - - if (bf_matcher_get_op(matcher) == BF_MATCHER_EQ) { - /* If we want to match an IP, both addr->data[0] and addr->data[1] - * must match the packet, otherwise we jump to the next rule. */ - EMIT(program, - BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr->data[7] & mask[7], - addr->data[6] & mask[6], - addr->data[5] & mask[5], - addr->data[4] & mask[4]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, - BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr->data[3] & mask[3], - addr->data[2] & mask[2], - addr->data[1] & mask[1], - addr->data[0] & mask[0]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_3, 0)); - - EMIT(program, - BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr->data[15] & mask[15], - addr->data[14] & mask[14], - addr->data[13] & mask[13], - addr->data[12] & mask[12]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, - BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr->data[11] & mask[11], - addr->data[10] & mask[10], - addr->data[9] & mask[9], - addr->data[8] & mask[8]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, 0)); - } else { - /* If we want to *not* match an IP, none of addr->data[0] and - * addr->data[1] should match the packet, otherwise we jump to the - * next rule. */ - EMIT(program, - BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr->data[7] & mask[7], - addr->data[6] & mask[6], - addr->data[5] & mask[5], - addr->data[4] & mask[4]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, - BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr->data[3] & mask[3], - addr->data[2] & mask[2], - addr->data[1] & mask[1], - addr->data[0] & mask[0]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - - /* Branching: - * - addr->data[0] matches the address' 64 MSB: continue to compare - * the address' 64 LSB. - * - addr->data[0] doesn't matches the address' 64 MSB: jump to the - * end of the matcher to continue the processing of the current rule. - * This matcher matched. */ - j0 = bf_jmpctx_get(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_3, 0)); - - EMIT(program, - BPF_MOV32_IMM(BPF_REG_3, _bf_make32(addr->data[15] & mask[15], - addr->data[14] & mask[14], - addr->data[13] & mask[13], - addr->data[12] & mask[12]))); - EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 32)); - EMIT(program, - BPF_MOV32_IMM(BPF_REG_4, _bf_make32(addr->data[11] & mask[11], - addr->data[10] & mask[10], - addr->data[9] & mask[9], - addr->data[8] & mask[8]))); - EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_3, BPF_REG_4)); - - /* Branching: - * - addr->data[1] matches the address' 64 LSB: addr->data matches the - * packet's address, meaning the matcher doesn't match. Jump to the - * next rule. - * - addr->data[1] doesn't matches the address' 64 LSB: the matcher - * matched: addr->data is not equal to the packet's address. Continue - * processing with the next matcher. */ - j1 = bf_jmpctx_get(program, - BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, 0)); - - EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP_A(0)); - - // j0 and j1 should jump here if they can't match the packet's IP. - bf_jmpctx_cleanup(&j0); - bf_jmpctx_cleanup(&j1); - } - - return 0; -} - -static int _bf_matcher_generate_ip6_nexthdr(struct bf_program *program, - const struct bf_matcher *matcher) -{ - const uint8_t ehdr = *(uint8_t *)bf_matcher_payload(matcher); - uint8_t eh_mask; - - if ((bf_matcher_get_op(matcher) != BF_MATCHER_EQ) && - (bf_matcher_get_op(matcher) != BF_MATCHER_NE)) - return -EINVAL; - - switch (ehdr) { - case IPPROTO_HOPOPTS: - case IPPROTO_ROUTING: - case IPPROTO_DSTOPTS: - case IPPROTO_FRAGMENT: - case IPPROTO_AH: - case IPPROTO_MH: - eh_mask = (BF_IPV6_EH_HOPOPTS(ehdr == IPPROTO_HOPOPTS) | - BF_IPV6_EH_ROUTING(ehdr == IPPROTO_ROUTING) | - BF_IPV6_EH_FRAGMENT(ehdr == IPPROTO_FRAGMENT) | - BF_IPV6_EH_AH(ehdr == IPPROTO_AH) | - BF_IPV6_EH_DSTOPTS(ehdr == IPPROTO_DSTOPTS) | - BF_IPV6_EH_MH(ehdr == IPPROTO_MH)); - EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, - BF_PROG_CTX_OFF(ipv6_eh))); - EMIT(program, BPF_ALU64_IMM(BPF_AND, BPF_REG_1, eh_mask)); - EMIT_FIXUP_JMP_NEXT_RULE( - program, BPF_JMP_IMM((bf_matcher_get_op(matcher) == BF_MATCHER_EQ) ? - BPF_JEQ : - BPF_JNE, - BPF_REG_1, 0, 0)); - break; - default: - /* check l4 protocols using BPF_REG_8 */ - EMIT_FIXUP_JMP_NEXT_RULE( - program, BPF_JMP_IMM((bf_matcher_get_op(matcher) == BF_MATCHER_EQ) ? - BPF_JNE : - BPF_JEQ, - BPF_REG_8, ehdr, 0)); - break; - } - - return 0; -} - -static int _bf_matcher_generate_ip6_dscp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint8_t dscp; - - assert(program); - assert(matcher); - - dscp = *(uint8_t *)bf_matcher_payload(matcher); - - /* IPv6 DSCP (traffic class) spans bits 4-11 of the header: - * Byte 0: version (4 bits) | dscp_high (4 bits) - * Byte 1: dscp_low (4 bits) | flow_label_high (4 bits) - * Load 2 bytes, mask with 0x0ff0, compare against dscp << 4. */ - - EMIT(program, BPF_LDX_MEM(BPF_H, BPF_REG_1, BPF_REG_6, 0)); - EMIT(program, BPF_ALU64_IMM(BPF_AND, BPF_REG_1, 0x0ff0)); - - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_IMM(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : - BPF_JEQ, - BPF_REG_1, (uint16_t)dscp << 4, 0)); - - return 0; -} - -int bf_matcher_generate_ip6(struct bf_program *program, - const struct bf_matcher *matcher) -{ - int r; - - EMIT_FIXUP_JMP_NEXT_RULE( - program, BPF_JMP_IMM(BPF_JNE, BPF_REG_7, htobe16(ETH_P_IPV6), 0)); - - EMIT(program, - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, BF_PROG_CTX_OFF(l3_hdr))); - - switch (bf_matcher_get_type(matcher)) { - case BF_MATCHER_IP6_SADDR: - case BF_MATCHER_IP6_DADDR: - r = _bf_matcher_generate_ip6_addr(program, matcher); - break; - case BF_MATCHER_IP6_SNET: - case BF_MATCHER_IP6_DNET: - r = _bf_matcher_generate_ip6_net(program, matcher); - break; - case BF_MATCHER_IP6_NEXTHDR: - r = _bf_matcher_generate_ip6_nexthdr(program, matcher); - break; - case BF_MATCHER_IP6_DSCP: - r = _bf_matcher_generate_ip6_dscp(program, matcher); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher type %d", - bf_matcher_get_type(matcher)); - }; - - return r; -} diff --git a/src/bpfilter/cgen/matcher/ip6.h b/src/bpfilter/cgen/matcher/ip6.h deleted file mode 100644 index afe2da292..000000000 --- a/src/bpfilter/cgen/matcher/ip6.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#pragma once - -struct bf_matcher; -struct bf_program; - -/** - * Generate the bytecode for the BF_MATCHER_IP6_* matcher types. - * - * @param program Program to generate the bytecode into. Can't be NULL. - * @param matcher Matcher to generate the bytecode for. Can't be NULL. - * @return 0 on success, negative errno value on failure. - */ -int bf_matcher_generate_ip6(struct bf_program *program, - const struct bf_matcher *matcher); diff --git a/src/bpfilter/cgen/matcher/meta.c b/src/bpfilter/cgen/matcher/meta.c index 719e3fb52..859bb4728 100644 --- a/src/bpfilter/cgen/matcher/meta.c +++ b/src/bpfilter/cgen/matcher/meta.c @@ -21,6 +21,7 @@ #include #include "cgen/elfstub.h" +#include "cgen/matcher/cmp.h" #include "cgen/program.h" #include "cgen/runtime.h" #include "cgen/swich.h" @@ -40,29 +41,6 @@ static int _bf_matcher_generate_meta_iface(struct bf_program *program, return 0; } -static int _bf_matcher_generate_meta_l3_proto(struct bf_program *program, - const struct bf_matcher *matcher) -{ - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_7, - htobe16(*(uint16_t *)bf_matcher_payload(matcher)), 0)); - - return 0; -} - -static int _bf_matcher_generate_meta_l4_proto(struct bf_program *program, - const struct bf_matcher *matcher) -{ - EMIT_FIXUP_JMP_NEXT_RULE( - program, - BPF_JMP_IMM(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : - BPF_JEQ, - BPF_REG_8, *(uint8_t *)bf_matcher_payload(matcher), 0)); - - return 0; -} - static int _bf_matcher_generate_meta_probability(struct bf_program *program, const struct bf_matcher *matcher) @@ -112,97 +90,13 @@ static int _bf_matcher_generate_meta_port(struct bf_program *program, // If r1 == 0: no TCP nor UDP header found, jump to the next rule EMIT_FIXUP_JMP_NEXT_RULE(program, BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 0)); - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, *port, 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, *port, 0)); - break; - case BF_MATCHER_RANGE: - /* Convert the big-endian value stored in the packet into a - * little-endian value for x86 and arm before comparing it to the - * reference value. This is a JLT/JGT comparison, we need to have the - * MSB where the machine expects then. */ + if (bf_matcher_get_op(matcher) == BF_MATCHER_RANGE) { EMIT(program, BPF_BSWAP(BPF_REG_1, 16)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JLT, BPF_REG_1, port[0], 0)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JGT, BPF_REG_1, port[1], 0)); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher operator '%s' (%d)", - bf_matcher_op_to_str(bf_matcher_get_op(matcher)), - bf_matcher_get_op(matcher)); + return bf_cmp_range(program, port[0], port[1], BPF_REG_1); } - return 0; -} - -int bf_matcher_generate_meta_mark_cmp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint32_t mark; - - assert(program); - assert(matcher); - - mark = *(uint32_t *)bf_matcher_payload(matcher); - - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, mark, 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, mark, 0)); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher operator '%s' (%d)", - bf_matcher_op_to_str(bf_matcher_get_op(matcher)), - bf_matcher_get_op(matcher)); - } - - return 0; -} - -int bf_matcher_generate_meta_flow_hash_cmp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint32_t *hash; - - assert(program); - assert(matcher); - - hash = (uint32_t *)bf_matcher_payload(matcher); - - EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_hash_recalc)); - - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_0, hash[0], 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, hash[0], 0)); - break; - case BF_MATCHER_RANGE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JLT, BPF_REG_0, hash[0], 0)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JGT, BPF_REG_0, hash[1], 0)); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher operator '%s' (%d)", - bf_matcher_op_to_str(bf_matcher_get_op(matcher)), - bf_matcher_get_op(matcher)); - } - - return 0; + return bf_cmp_value(program, bf_matcher_get_op(matcher), port, 2, + BPF_REG_1); } static int @@ -257,28 +151,23 @@ _bf_matcher_generate_meta_flow_probability(struct bf_program *program, int bf_matcher_generate_meta(struct bf_program *program, const struct bf_matcher *matcher) { - int r; - switch (bf_matcher_get_type(matcher)) { case BF_MATCHER_META_IFACE: - r = _bf_matcher_generate_meta_iface(program, matcher); - break; - case BF_MATCHER_META_L3_PROTO: - r = _bf_matcher_generate_meta_l3_proto(program, matcher); - break; + return _bf_matcher_generate_meta_iface(program, matcher); + case BF_MATCHER_META_L3_PROTO: { + uint16_t be_val = htobe16(*(uint16_t *)bf_matcher_payload(matcher)); + return bf_cmp_value(program, BF_MATCHER_EQ, &be_val, 2, BPF_REG_7); + } case BF_MATCHER_META_L4_PROTO: - r = _bf_matcher_generate_meta_l4_proto(program, matcher); - break; + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 1, BPF_REG_8); case BF_MATCHER_META_PROBABILITY: - r = _bf_matcher_generate_meta_probability(program, matcher); - break; + return _bf_matcher_generate_meta_probability(program, matcher); case BF_MATCHER_META_SPORT: case BF_MATCHER_META_DPORT: - r = _bf_matcher_generate_meta_port(program, matcher); - break; + return _bf_matcher_generate_meta_port(program, matcher); case BF_MATCHER_META_FLOW_PROBABILITY: - r = _bf_matcher_generate_meta_flow_probability(program, matcher); - break; + return _bf_matcher_generate_meta_flow_probability(program, matcher); case BF_MATCHER_META_MARK: case BF_MATCHER_META_FLOW_HASH: return bf_err_r(-ENOTSUP, @@ -287,10 +176,5 @@ int bf_matcher_generate_meta(struct bf_program *program, default: return bf_err_r(-EINVAL, "unknown matcher type %d", bf_matcher_get_type(matcher)); - }; - - if (r) - return r; - - return 0; + } } diff --git a/src/bpfilter/cgen/matcher/meta.h b/src/bpfilter/cgen/matcher/meta.h index 7594c92c3..14791c923 100644 --- a/src/bpfilter/cgen/matcher/meta.h +++ b/src/bpfilter/cgen/matcher/meta.h @@ -10,29 +10,3 @@ struct bf_program; int bf_matcher_generate_meta(struct bf_program *program, const struct bf_matcher *matcher); - -/** - * @brief Generate bytecode to compare a packet mark. - * - * The mark value must already be loaded into `BPF_REG_1`. - * - * @param program Program being generated. Can't be NULL. - * @param matcher Matcher to generate comparison for. Can't be NULL. - * @return 0 on success, negative errno on error. - */ -int bf_matcher_generate_meta_mark_cmp(struct bf_program *program, - const struct bf_matcher *matcher); - -/** - * @brief Generate bytecode to compute and compare a flow hash. - * - * The skb pointer must already be loaded into `BPF_REG_1`. Calls - * `bpf_get_hash_recalc` on the skb, then compares the result against the - * matcher's payload. - * - * @param program Program being generated. Can't be NULL. - * @param matcher Matcher to generate comparison for. Can't be NULL. - * @return 0 on success, negative errno on error. - */ -int bf_matcher_generate_meta_flow_hash_cmp(struct bf_program *program, - const struct bf_matcher *matcher); diff --git a/src/bpfilter/cgen/matcher/packet.c b/src/bpfilter/cgen/matcher/packet.c index ece0a0089..0d4ed72b5 100644 --- a/src/bpfilter/cgen/matcher/packet.c +++ b/src/bpfilter/cgen/matcher/packet.c @@ -5,27 +5,280 @@ #include "cgen/matcher/packet.h" +#include +#include +#include // NOLINT +#include + +#include #include +#include #include #include #include -#include "cgen/matcher/icmp.h" -#include "cgen/matcher/ip4.h" -#include "cgen/matcher/ip6.h" +#include "cgen/matcher/cmp.h" #include "cgen/matcher/meta.h" #include "cgen/matcher/set.h" -#include "cgen/matcher/tcp.h" -#include "cgen/matcher/udp.h" #include "cgen/program.h" +#include "cgen/stub.h" +#include "filter.h" + +/** + * Packet matcher codegen follows a three-stage pipeline: + * + * 1. Protocol check: `_bf_program_generate_rule()` (in program.c) + * emits deduplicated protocol guards before the matcher loop, + * so each L3/L4 protocol is verified at most once per rule. + * + * 2. Header + field load: `bf_stub_load_header()` loads the header + * base address into `R6`, then `_bf_matcher_pkt_load_field()` reads + * the target field into the specified register (and `reg+1` for + * 128-bit values such as IPv6 addresses). + * + * 3. Comparison: A `bf_cmp_*` function compares the value in the + * specified register against the matcher's reference payload. + */ + +#define BF_IPV6_EH_HOPOPTS(x) ((x) << 0) +#define BF_IPV6_EH_ROUTING(x) ((x) << 1) +#define BF_IPV6_EH_FRAGMENT(x) ((x) << 2) +#define BF_IPV6_EH_AH(x) ((x) << 3) +#define BF_IPV6_EH_DSTOPTS(x) ((x) << 4) +#define BF_IPV6_EH_MH(x) ((x) << 5) + +/** + * @brief Load a packet field from the header into the specified register. + * + * `R6` must already point to the header base. For 128-bit fields (IPv6 + * addresses), the low 8 bytes are loaded into `reg` and the high 8 bytes into + * `reg + 1`. + * + * @param program Program to emit into. Can't be NULL. + * @param meta Matcher metadata describing field offset and size. Can't be NULL. + * @param reg BPF register to load the value into. + * @return 0 on success, negative errno on error. + */ +static int _bf_matcher_pkt_load_field(struct bf_program *program, + const struct bf_matcher_meta *meta, + int reg) +{ + switch (meta->hdr_payload_size) { + case 1: + EMIT(program, + BPF_LDX_MEM(BPF_B, reg, BPF_REG_6, meta->hdr_payload_offset)); + break; + case 2: + EMIT(program, + BPF_LDX_MEM(BPF_H, reg, BPF_REG_6, meta->hdr_payload_offset)); + break; + case 4: + EMIT(program, + BPF_LDX_MEM(BPF_W, reg, BPF_REG_6, meta->hdr_payload_offset)); + break; + case 8: + EMIT(program, + BPF_LDX_MEM(BPF_DW, reg, BPF_REG_6, meta->hdr_payload_offset)); + break; + case 16: + EMIT(program, + BPF_LDX_MEM(BPF_DW, reg, BPF_REG_6, meta->hdr_payload_offset)); + EMIT(program, BPF_LDX_MEM(BPF_DW, reg + 1, BPF_REG_6, + meta->hdr_payload_offset + 8)); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int _bf_matcher_pkt_load(struct bf_program *program, + const struct bf_matcher_meta *meta, int reg) +{ + int r; + + r = bf_stub_load_header(program, meta, BPF_REG_6); + if (r) + return r; + + return _bf_matcher_pkt_load_field(program, meta, reg); +} + +/** + * @brief Generic load + value compare for matchers whose field size and offset + * are fully described by `_bf_matcher_metas`. + * + * Emits: header load -> field load -> `bf_cmp_value`. + * + * @param program Program to generate bytecode into. Can't be NULL. + * @param matcher Matcher to generate bytecode for. Can't be NULL. + * @param meta Matcher metadata describing field size and offset. Can't be NULL. + * @return 0 on success, negative errno on error. + */ +static int _bf_matcher_pkt_load_and_cmp(struct bf_program *program, + const struct bf_matcher *matcher, + const struct bf_matcher_meta *meta) +{ + int r; + + r = _bf_matcher_pkt_load(program, meta, BPF_REG_1); + if (r) + return r; + + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), meta->hdr_payload_size, + BPF_REG_1); +} + +static int _bf_matcher_pkt_generate_net(struct bf_program *program, + const struct bf_matcher *matcher, + const struct bf_matcher_meta *meta) +{ + const uint32_t prefixlen = *(const uint32_t *)bf_matcher_payload(matcher); + const void *data = + (const uint8_t *)bf_matcher_payload(matcher) + sizeof(uint32_t); + int r; + + r = _bf_matcher_pkt_load(program, meta, BPF_REG_1); + if (r) + return r; + + return bf_cmp_masked_value(program, bf_matcher_get_op(matcher), data, + prefixlen, meta->hdr_payload_size, BPF_REG_1); +} + +static int _bf_matcher_pkt_generate_port(struct bf_program *program, + const struct bf_matcher *matcher, + const struct bf_matcher_meta *meta) +{ + int r; + + r = _bf_matcher_pkt_load(program, meta, BPF_REG_1); + if (r) + return r; + + if (bf_matcher_get_op(matcher) == BF_MATCHER_RANGE) { + uint16_t *ports = (uint16_t *)bf_matcher_payload(matcher); + /* Convert the big-endian value stored in the packet into a + * little-endian value for x86 and arm before comparing it to the + * reference value. This is a JLT/JGT comparison, we need to have the + * MSB where the machine expects then. */ + EMIT(program, BPF_BSWAP(BPF_REG_1, 16)); + return bf_cmp_range(program, ports[0], ports[1], BPF_REG_1); + } + + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), meta->hdr_payload_size, + BPF_REG_1); +} + +static int +_bf_matcher_pkt_generate_tcp_flags(struct bf_program *program, + const struct bf_matcher *matcher, + const struct bf_matcher_meta *meta) +{ + int r; + + r = _bf_matcher_pkt_load(program, meta, BPF_REG_1); + if (r) + return r; + + switch (bf_matcher_get_op(matcher)) { + case BF_MATCHER_ANY: + case BF_MATCHER_ALL: + return bf_cmp_bitfield(program, bf_matcher_get_op(matcher), + *(uint8_t *)bf_matcher_payload(matcher), + BPF_REG_1); + default: + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), meta->hdr_payload_size, + BPF_REG_1); + } +} + +static int +_bf_matcher_pkt_generate_ip6_nexthdr(struct bf_program *program, + const struct bf_matcher *matcher) +{ + const uint8_t ehdr = *(uint8_t *)bf_matcher_payload(matcher); + uint8_t eh_mask; + + switch (ehdr) { + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: + case IPPROTO_DSTOPTS: + case IPPROTO_FRAGMENT: + case IPPROTO_AH: + case IPPROTO_MH: + eh_mask = (BF_IPV6_EH_HOPOPTS(ehdr == IPPROTO_HOPOPTS) | + BF_IPV6_EH_ROUTING(ehdr == IPPROTO_ROUTING) | + BF_IPV6_EH_FRAGMENT(ehdr == IPPROTO_FRAGMENT) | + BF_IPV6_EH_AH(ehdr == IPPROTO_AH) | + BF_IPV6_EH_DSTOPTS(ehdr == IPPROTO_DSTOPTS) | + BF_IPV6_EH_MH(ehdr == IPPROTO_MH)); + EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, + BF_PROG_CTX_OFF(ipv6_eh))); + EMIT(program, BPF_ALU64_IMM(BPF_AND, BPF_REG_1, eh_mask)); + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_IMM((bf_matcher_get_op(matcher) == BF_MATCHER_EQ) ? + BPF_JEQ : + BPF_JNE, + BPF_REG_1, 0, 0)); + break; + default: + /* check l4 protocols using `BPF_REG_8` */ + EMIT_FIXUP_JMP_NEXT_RULE( + program, BPF_JMP_IMM((bf_matcher_get_op(matcher) == BF_MATCHER_EQ) ? + BPF_JNE : + BPF_JEQ, + BPF_REG_8, ehdr, 0)); + break; + } + + return 0; +} + +static int _bf_matcher_pkt_generate_ip6_dscp(struct bf_program *program, + const struct bf_matcher *matcher, + const struct bf_matcher_meta *meta) +{ + uint8_t dscp; + int r; + + r = bf_stub_load_header(program, meta, BPF_REG_6); + if (r) + return r; + + dscp = *(uint8_t *)bf_matcher_payload(matcher); + + /* IPv6 DSCP (traffic class) spans bits 4-11 of the header: + * Byte 0: version (4 bits) | dscp_high (4 bits) + * Byte 1: dscp_low (4 bits) | flow_label_high (4 bits) + * Load 2 bytes, mask with 0x0ff0, compare against dscp << 4. */ + EMIT(program, BPF_LDX_MEM(BPF_H, BPF_REG_1, BPF_REG_6, 0)); + EMIT(program, BPF_ALU64_IMM(BPF_AND, BPF_REG_1, 0x0ff0)); + + EMIT_FIXUP_JMP_NEXT_RULE( + program, + BPF_JMP_IMM(bf_matcher_get_op(matcher) == BF_MATCHER_EQ ? BPF_JNE : + BPF_JEQ, + BPF_REG_1, (uint16_t)dscp << 4, 0)); + + return 0; +} int bf_matcher_generate_packet(struct bf_program *program, const struct bf_matcher *matcher) { + const struct bf_matcher_meta *meta; + assert(program); assert(matcher); + meta = bf_matcher_get_meta(bf_matcher_get_type(matcher)); + switch (bf_matcher_get_type(matcher)) { case BF_MATCHER_META_IFACE: case BF_MATCHER_META_L3_PROTO: @@ -41,31 +294,32 @@ int bf_matcher_generate_packet(struct bf_program *program, "matcher '%s' is not supported by this flavor", bf_matcher_type_to_str(bf_matcher_get_type(matcher))); case BF_MATCHER_IP4_SADDR: - case BF_MATCHER_IP4_SNET: case BF_MATCHER_IP4_DADDR: - case BF_MATCHER_IP4_DNET: case BF_MATCHER_IP4_PROTO: case BF_MATCHER_IP4_DSCP: - return bf_matcher_generate_ip4(program, matcher); case BF_MATCHER_IP6_SADDR: - case BF_MATCHER_IP6_SNET: case BF_MATCHER_IP6_DADDR: + case BF_MATCHER_ICMP_TYPE: + case BF_MATCHER_ICMP_CODE: + case BF_MATCHER_ICMPV6_TYPE: + case BF_MATCHER_ICMPV6_CODE: + return _bf_matcher_pkt_load_and_cmp(program, matcher, meta); + case BF_MATCHER_IP4_SNET: + case BF_MATCHER_IP4_DNET: + case BF_MATCHER_IP6_SNET: case BF_MATCHER_IP6_DNET: - case BF_MATCHER_IP6_NEXTHDR: - case BF_MATCHER_IP6_DSCP: - return bf_matcher_generate_ip6(program, matcher); + return _bf_matcher_pkt_generate_net(program, matcher, meta); case BF_MATCHER_TCP_SPORT: case BF_MATCHER_TCP_DPORT: - case BF_MATCHER_TCP_FLAGS: - return bf_matcher_generate_tcp(program, matcher); case BF_MATCHER_UDP_SPORT: case BF_MATCHER_UDP_DPORT: - return bf_matcher_generate_udp(program, matcher); - case BF_MATCHER_ICMP_TYPE: - case BF_MATCHER_ICMP_CODE: - case BF_MATCHER_ICMPV6_TYPE: - case BF_MATCHER_ICMPV6_CODE: - return bf_matcher_generate_icmp(program, matcher); + return _bf_matcher_pkt_generate_port(program, matcher, meta); + case BF_MATCHER_TCP_FLAGS: + return _bf_matcher_pkt_generate_tcp_flags(program, matcher, meta); + case BF_MATCHER_IP6_NEXTHDR: + return _bf_matcher_pkt_generate_ip6_nexthdr(program, matcher); + case BF_MATCHER_IP6_DSCP: + return _bf_matcher_pkt_generate_ip6_dscp(program, matcher, meta); case BF_MATCHER_SET: return bf_matcher_generate_set(program, matcher); default: diff --git a/src/bpfilter/cgen/matcher/tcp.c b/src/bpfilter/cgen/matcher/tcp.c deleted file mode 100644 index 709209105..000000000 --- a/src/bpfilter/cgen/matcher/tcp.c +++ /dev/null @@ -1,121 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#include "cgen/matcher/tcp.h" - -#include -#include -#include // NOLINT -#include - -#include -#include -#include -#include - -#include -#include - -#include "cgen/program.h" -#include "filter.h" - -static int _bf_matcher_generate_tcp_port(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint16_t *port = (uint16_t *)bf_matcher_payload(matcher); - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_TCP_SPORT ? - offsetof(struct tcphdr, source) : - offsetof(struct tcphdr, dest); - - EMIT(program, BPF_LDX_MEM(BPF_H, BPF_REG_1, BPF_REG_6, offset)); - - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, *port, 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, *port, 0)); - break; - case BF_MATCHER_RANGE: - /* Convert the big-endian value stored in the packet into a - * little-endian value for x86 and arm before comparing it to the - * reference value. This is a JLT/JGT comparison, we need to have the - * MSB where the machine expects then. */ - EMIT(program, BPF_BSWAP(BPF_REG_1, 16)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JLT, BPF_REG_1, port[0], 0)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JGT, BPF_REG_1, port[1], 0)); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher operator '%s' (%d)", - bf_matcher_op_to_str(bf_matcher_get_op(matcher)), - bf_matcher_get_op(matcher)); - } - - return 0; -} - -static int _bf_matcher_generate_tcp_flags(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint8_t flags = *(uint8_t *)bf_matcher_payload(matcher); - - EMIT(program, BPF_LDX_MEM(BPF_B, BPF_REG_1, BPF_REG_6, 13)); - - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, flags, 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, flags, 0)); - break; - case BF_MATCHER_ANY: - EMIT(program, BPF_ALU32_IMM(BPF_AND, BPF_REG_1, flags)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 0)); - break; - case BF_MATCHER_ALL: - EMIT(program, BPF_ALU32_IMM(BPF_AND, BPF_REG_1, flags)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, flags, 0)); - break; - default: - return bf_err_r(-EINVAL, "unsupported matcher for tcp.flags: %s", - bf_matcher_op_to_str(bf_matcher_get_op(matcher))); - } - - return 0; -} - -int bf_matcher_generate_tcp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - int r; - - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_8, IPPROTO_TCP, 0)); - EMIT(program, - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, BF_PROG_CTX_OFF(l4_hdr))); - - switch (bf_matcher_get_type(matcher)) { - case BF_MATCHER_TCP_SPORT: - case BF_MATCHER_TCP_DPORT: - r = _bf_matcher_generate_tcp_port(program, matcher); - break; - case BF_MATCHER_TCP_FLAGS: - r = _bf_matcher_generate_tcp_flags(program, matcher); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher type %d", - bf_matcher_get_type(matcher)); - }; - - return r; -} diff --git a/src/bpfilter/cgen/matcher/tcp.h b/src/bpfilter/cgen/matcher/tcp.h deleted file mode 100644 index ec07ac328..000000000 --- a/src/bpfilter/cgen/matcher/tcp.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#pragma once - -struct bf_matcher; -struct bf_program; - -int bf_matcher_generate_tcp(struct bf_program *program, - const struct bf_matcher *matcher); diff --git a/src/bpfilter/cgen/matcher/udp.c b/src/bpfilter/cgen/matcher/udp.c deleted file mode 100644 index 2eb46b71a..000000000 --- a/src/bpfilter/cgen/matcher/udp.c +++ /dev/null @@ -1,84 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#include "cgen/matcher/udp.h" - -#include -#include -#include // NOLINT -#include - -#include -#include -#include -#include - -#include -#include - -#include "cgen/program.h" -#include "filter.h" - -static int _bf_matcher_generate_udp_port(struct bf_program *program, - const struct bf_matcher *matcher) -{ - uint16_t *port = (uint16_t *)bf_matcher_payload(matcher); - size_t offset = bf_matcher_get_type(matcher) == BF_MATCHER_UDP_SPORT ? - offsetof(struct udphdr, source) : - offsetof(struct udphdr, dest); - - EMIT(program, BPF_LDX_MEM(BPF_H, BPF_REG_1, BPF_REG_6, offset)); - - switch (bf_matcher_get_op(matcher)) { - case BF_MATCHER_EQ: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_1, *port, 0)); - break; - case BF_MATCHER_NE: - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, *port, 0)); - break; - case BF_MATCHER_RANGE: - /* Convert the big-endian value stored in the packet into a - * little-endian value for x86 and arm before comparing it to the - * reference value. This is a JLT/JGT comparison, we need to have the - * MSB where the machine expects then. */ - EMIT(program, BPF_BSWAP(BPF_REG_1, 16)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JLT, BPF_REG_1, port[0], 0)); - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JGT, BPF_REG_1, port[1], 0)); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher operator '%s' (%d)", - bf_matcher_op_to_str(bf_matcher_get_op(matcher)), - bf_matcher_get_op(matcher)); - } - - return 0; -} - -int bf_matcher_generate_udp(struct bf_program *program, - const struct bf_matcher *matcher) -{ - int r; - - EMIT_FIXUP_JMP_NEXT_RULE(program, - BPF_JMP_IMM(BPF_JNE, BPF_REG_8, IPPROTO_UDP, 0)); - EMIT(program, - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, BF_PROG_CTX_OFF(l4_hdr))); - - switch (bf_matcher_get_type(matcher)) { - case BF_MATCHER_UDP_SPORT: - case BF_MATCHER_UDP_DPORT: - r = _bf_matcher_generate_udp_port(program, matcher); - break; - default: - return bf_err_r(-EINVAL, "unknown matcher type %d", - bf_matcher_get_type(matcher)); - }; - - return r; -} diff --git a/src/bpfilter/cgen/matcher/udp.h b/src/bpfilter/cgen/matcher/udp.h deleted file mode 100644 index c87190ba2..000000000 --- a/src/bpfilter/cgen/matcher/udp.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - */ - -#pragma once - -struct bf_matcher; -struct bf_program; - -int bf_matcher_generate_udp(struct bf_program *program, - const struct bf_matcher *matcher); diff --git a/src/bpfilter/cgen/nf.c b/src/bpfilter/cgen/nf.c index 9f3984d19..103883c73 100644 --- a/src/bpfilter/cgen/nf.c +++ b/src/bpfilter/cgen/nf.c @@ -23,7 +23,7 @@ #include #include "cgen/jmp.h" -#include "cgen/matcher/meta.h" +#include "cgen/matcher/cmp.h" #include "cgen/matcher/packet.h" #include "cgen/program.h" #include "cgen/stub.h" @@ -146,15 +146,23 @@ static int _bf_nf_gen_inline_matcher(struct bf_program *program, return offset; EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, offset)); - return bf_matcher_generate_meta_mark_cmp(program, matcher); + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 4, BPF_REG_1); case BF_MATCHER_META_FLOW_HASH: EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg))); if ((offset = bf_btf_get_field_off("bpf_nf_ctx", "skb")) < 0) return offset; EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, offset)); + EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_hash_recalc)); - return bf_matcher_generate_meta_flow_hash_cmp(program, matcher); + if (bf_matcher_get_op(matcher) == BF_MATCHER_RANGE) { + uint32_t *hash = (uint32_t *)bf_matcher_payload(matcher); + return bf_cmp_range(program, hash[0], hash[1], BPF_REG_0); + } + + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 4, BPF_REG_0); default: return bf_matcher_generate_packet(program, matcher); } diff --git a/src/bpfilter/cgen/program.c b/src/bpfilter/cgen/program.c index 9f98f64e0..1cd6c6596 100644 --- a/src/bpfilter/cgen/program.c +++ b/src/bpfilter/cgen/program.c @@ -41,7 +41,6 @@ #include "cgen/fixup.h" #include "cgen/handle.h" #include "cgen/jmp.h" -#include "cgen/matcher/packet.h" #include "cgen/nf.h" #include "cgen/printer.h" #include "cgen/prog/link.h" @@ -285,15 +284,41 @@ static int _bf_program_fixup(struct bf_program *program, static int _bf_program_generate_rule(struct bf_program *program, struct bf_rule *rule) { + uint32_t checked_layers = 0; int r; assert(program); assert(rule); + assert(program->runtime.ops->gen_inline_matcher); if (rule->disabled) return 0; - assert(program->runtime.ops->gen_inline_matcher); + bf_list_foreach (&rule->matchers, matcher_node) { + struct bf_matcher *matcher = bf_list_node_get_data(matcher_node); + const struct bf_matcher_meta *meta = + bf_matcher_get_meta(bf_matcher_get_type(matcher)); + + if (bf_matcher_get_type(matcher) == BF_MATCHER_SET) + continue; + + if (!meta) { + return bf_err_r(-EINVAL, "missing meta for matcher type %d", + bf_matcher_get_type(matcher)); + } + + if (checked_layers & BF_FLAG(meta->layer)) + continue; + + if (meta->layer == BF_MATCHER_LAYER_2 || + meta->layer == BF_MATCHER_LAYER_3 || + meta->layer == BF_MATCHER_LAYER_4) { + r = bf_stub_rule_check_protocol(program, meta); + if (r) + return r; + checked_layers |= BF_FLAG(meta->layer); + } + } bf_list_foreach (&rule->matchers, matcher_node) { struct bf_matcher *matcher = bf_list_node_get_data(matcher_node); diff --git a/src/bpfilter/cgen/tc.c b/src/bpfilter/cgen/tc.c index 077f8d01a..d83fcd411 100644 --- a/src/bpfilter/cgen/tc.c +++ b/src/bpfilter/cgen/tc.c @@ -19,7 +19,7 @@ #include #include "cgen/cgen.h" -#include "cgen/matcher/meta.h" +#include "cgen/matcher/cmp.h" #include "cgen/matcher/packet.h" #include "cgen/program.h" #include "cgen/stub.h" @@ -101,12 +101,20 @@ static int _bf_tc_gen_inline_matcher(struct bf_program *program, EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, offsetof(struct __sk_buff, mark))); - return bf_matcher_generate_meta_mark_cmp(program, matcher); + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 4, BPF_REG_1); case BF_MATCHER_META_FLOW_HASH: EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg))); + EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_hash_recalc)); - return bf_matcher_generate_meta_flow_hash_cmp(program, matcher); + if (bf_matcher_get_op(matcher) == BF_MATCHER_RANGE) { + uint32_t *hash = (uint32_t *)bf_matcher_payload(matcher); + return bf_cmp_range(program, hash[0], hash[1], BPF_REG_0); + } + + return bf_cmp_value(program, bf_matcher_get_op(matcher), + bf_matcher_payload(matcher), 4, BPF_REG_0); default: return bf_matcher_generate_packet(program, matcher); }