fix(modern_bpf): bpf_loop the sendmmsg/recvmmsg iovec data store

driver/modern_bpf/helpers/store/auxmap_store_params.h

@@ -1357,6 +1357,145 @@ static __always_inline void auxmap__store_iovec_data_param(struct auxiliary_map
 	}
 }
 
+/* bpf_loop()-based variant of the native iovec data store, for use inside the
+ * sendmmsg_x/recvmmsg_x bpf_loop callbacks.
+ *
+ * The plain for(j < MAX_IOVCNT) loop in auxmap__store_iovec_data_param_64 is a
+ * bounded loop. When it sits inside a bpf_loop callback, stricter kernel
+ * verifiers re-explore the whole 32-iteration loop on every fixed-point pass of
+ * the outer callback, multiplying processed instructions. Replacing it with a
+ * bpf_loop helper makes the verifier check the per-iteration body once instead,
+ * collapsing that multiplication and keeping the program under the
+ * processed-instruction limit.
+ *
+ * The loop callback cannot carry a map-value (auxmap) pointer through the
+ * bpf_loop context, so it re-fetches the per-cpu auxmap via auxmap__get() and
+ * re-derives the iovec scratch area. Mutable loop state lives in the context. */
+typedef struct {
+	unsigned long iov_cnt;
+	unsigned long len_to_read;
+	unsigned long total_size_to_read;
+	bool truncated;
+} iovec_data_loop_ctx_t;
+
+static long iovec_data_loop_callback(uint32_t index, void *ctx) {
+	iovec_data_loop_ctx_t *c = (iovec_data_loop_ctx_t *)ctx;
+
+	if(c->total_size_to_read > c->len_to_read) {
+		c->total_size_to_read = c->len_to_read;
+		return 1;
+	}
+	if(index >= c->iov_cnt) {
+		return 1;
+	}
+
+	struct auxiliary_map *auxmap = auxmap__get();
+	if(!auxmap) {
+		return 1;
+	}
+
+	/* Mask the index so the verifier can bound the map-value access. */
+	uint32_t j = index & (MAX_IOVCNT - 1);
+	const struct iovec *iovec = (const struct iovec *)&auxmap->data[MAX_PARAM_SIZE];
+
+	uint16_t bytes_read = push__bytebuf(auxmap->data,
+	                                    &auxmap->payload_pos,
+	                                    (unsigned long)iovec[j].iov_base,
+	                                    iovec[j].iov_len,
+	                                    USER);
+	if(!bytes_read) {
+		c->truncated = true;
+		return 1;
+	}
+	c->total_size_to_read += bytes_read;
+	return 0;
+}
+
+static __noinline void auxmap__store_iovec_data_param_64_bpf_loop(struct auxiliary_map *auxmap,
+                                                                  unsigned long iov_pointer,
+                                                                  unsigned long iov_cnt,
+                                                                  unsigned long len_to_read) {
+	/* We use the second part of our auxmap as a scratch space. */
+	unsigned long total_iovec_size = iov_cnt * bpf_core_type_size(struct iovec);
+
+	if(bpf_probe_read_user((void *)&auxmap->data[MAX_PARAM_SIZE],
+	                       SAFE_ACCESS(total_iovec_size),
+	                       (void *)iov_pointer)) {
+		/* in case of NULL iovec vector we return an empty param */
+		push__param_len(auxmap->data, &auxmap->lengths_pos, 0);
+		return;
+	}
+
+	uint64_t initial_payload_pos = auxmap->payload_pos;
+	iovec_data_loop_ctx_t ctx = {
+	        .iov_cnt = iov_cnt,
+	        .len_to_read = len_to_read,
+	        .total_size_to_read = 0,
+	        .truncated = false,
+	};
+
+	bpf_loop(MAX_IOVCNT, iovec_data_loop_callback, &ctx, 0);
+
+	/* Same len enforcement as the for-loop version: when we did not stop early
+	 * on a failed read, clamp payload_pos so we never exceed len_to_read. */
+	if(!ctx.truncated) {
+		auxmap->payload_pos = initial_payload_pos + ctx.total_size_to_read;
+	}
+	push__param_len(auxmap->data, &auxmap->lengths_pos, ctx.total_size_to_read);
+}
+
+/* Native bpf_loop iovec store wrapped in a __noinline boundary. handle_exit()
+ * for sendmmsg_x/recvmmsg_x runs as a bpf_loop callback, and stricter verifiers
+ * re-explore everything INLINED into that callback on every SCC fixed-point
+ * pass. Keeping the ia32 dispatch and the bpf_loop call behind a __noinline
+ * function means they are verified once instead of being multiplied by that
+ * re-exploration — which is what keeps the program under the 1M
+ * processed-instruction limit. */
+static __noinline void auxmap__store_iovec_data_param_mmsg_bpf_loop(struct auxiliary_map *auxmap,
+                                                                    unsigned long iov_pointer,
+                                                                    unsigned long iov_cnt,
+                                                                    unsigned long len_to_read) {
+	if(bpf_in_ia32_syscall()) {
+		/* ia32 (compat) path keeps the bounded for-loop (rare; compat layout). */
+		auxmap__store_iovec_data_param_32(auxmap, iov_pointer, iov_cnt, len_to_read);
+	} else {
+		auxmap__store_iovec_data_param_64_bpf_loop(auxmap, iov_pointer, iov_cnt, len_to_read);
+	}
+}
+
+/* Defined later in this header; used by the selector below for the legacy path. */
+static __noinline void auxmap__store_iovec_data_param_noinline(struct auxiliary_map *auxmap,
+                                                               unsigned long iov_pointer,
+                                                               unsigned long iov_cnt,
+                                                               unsigned long len_to_read);
+
+/* mmsg iovec data store selector. handle_exit() is shared by the bpf_loop
+ * programs (sendmmsg_x/recvmmsg_x, loaded only when the bpf_loop helper is
+ * available) and the legacy single-message programs
+ * (sendmmsg_old_x/recvmmsg_old_x, loaded when it is NOT).
+ *
+ * `use_bpf_loop` is a compile-time constant supplied by each caller, NOT a CO-RE
+ * feature probe: some kernels expose BPF_FUNC_loop in their BTF enum without
+ * implementing the helper, so bpf_core_enum_value_exists() would wrongly route
+ * the legacy program through bpf_loop and fail to load with "invalid func
+ * unknown#181". Because the constant is propagated through this __always_inline
+ * selector, it collapses to a single call to the matching __noinline store: the
+ * legacy program references only the for-loop store (never the bpf_loop helper,
+ * independent of BTF), while the bpf_loop program keeps the __noinline boundary
+ * that bounds its verifier work. The legacy path handles a single message and is
+ * not subject to the SCC blowup, so the for-loop store is correct there. */
+static __always_inline void auxmap__store_iovec_data_param_mmsg(struct auxiliary_map *auxmap,
+                                                                unsigned long iov_pointer,
+                                                                unsigned long iov_cnt,
+                                                                unsigned long len_to_read,
+                                                                bool use_bpf_loop) {
+	if(use_bpf_loop) {
+		auxmap__store_iovec_data_param_mmsg_bpf_loop(auxmap, iov_pointer, iov_cnt, len_to_read);
+	} else {
+		auxmap__store_iovec_data_param_noinline(auxmap, iov_pointer, iov_cnt, len_to_read);
+	}
+}
+
 /**
  * @brief Store the size extracted from a `user_msghdr` struct.
  * Please note: the size is an unsigned 32 bit value so

driver/modern_bpf/programs/tail_called/events/syscall_dispatched_events/recvmmsg.bpf.c

@@ -16,9 +16,9 @@ typedef struct {
 	struct mmsghdr *mmh;
 } recvmmsg_data_t;
 
-static __always_inline long handle_exit(uint32_t index, void *ctx) {
-	recvmmsg_data_t *data = (recvmmsg_data_t *)ctx;
-
+static __always_inline long recvmmsg_handle_msg(uint32_t index,
+                                                recvmmsg_data_t *data,
+                                                bool use_bpf_loop) {
 	/* Read individual fields instead of the full struct mmsghdr to keep
 	 * frame 1 stack small enough for the 512-byte 3-frame limit.
 	 * Handle both native and compat (ia32) layouts.
@@ -73,7 +73,7 @@ static __always_inline long handle_exit(uint32_t index, void *ctx) {
 	}
 
 	/* Parameter 4: data (type: PT_BYTEBUF) */
-	auxmap__store_iovec_data_param_noinline(auxmap, msg_iov, msg_iovlen, snaplen);
+	auxmap__store_iovec_data_param_mmsg(auxmap, msg_iov, msg_iovlen, snaplen, use_bpf_loop);
 
 	/* Parameter 5: tuple (type: PT_SOCKTUPLE) */
 	auxmap__store_socktuple_param_noinline(auxmap, data->fd, INBOUND, msg_name);
@@ -95,6 +95,12 @@ static __always_inline long handle_exit(uint32_t index, void *ctx) {
 	return 0;
 }
 
+/* bpf_loop callback used by recvmmsg_x. That program is loaded only when the
+ * bpf_loop helper is available, so the per-message iovec store uses bpf_loop. */
+static long handle_exit(uint32_t index, void *ctx) {
+	return recvmmsg_handle_msg(index, (recvmmsg_data_t *)ctx, true);
+}
+
 SEC("tp_btf/sys_exit")
 int BPF_PROG(recvmmsg_x, struct pt_regs *regs, long ret) {
 	if(ret <= 0) {
@@ -189,8 +195,8 @@ int BPF_PROG(recvmmsg_old_x, struct pt_regs *regs, long ret) {
 	        .mmh = (struct mmsghdr *)args[1],
 	};
 
-	// Send only first message
-	handle_exit(0, &data);
+	// Send only first message; legacy program runs without bpf_loop.
+	recvmmsg_handle_msg(0, &data, false);
 
 	return 0;
 }

driver/modern_bpf/programs/tail_called/events/syscall_dispatched_events/sendmmsg.bpf.c

@@ -16,9 +16,9 @@ typedef struct {
 	struct mmsghdr *mmh;
 } sendmmsg_exit_t;
 
-static __always_inline long handle_exit(uint32_t index, void *ctx) {
-	sendmmsg_exit_t *data = (sendmmsg_exit_t *)ctx;
-
+static __always_inline long sendmmsg_handle_msg(uint32_t index,
+                                                sendmmsg_exit_t *data,
+                                                bool use_bpf_loop) {
 	/* Read individual fields instead of the full struct mmsghdr to keep
 	 * frame 1 stack small enough for the 512-byte 3-frame limit.
 	 * Handle both native and compat (ia32) layouts.
@@ -67,7 +67,7 @@ static __always_inline long handle_exit(uint32_t index, void *ctx) {
 	}
 
 	/* Parameter 4: data (type: PT_BYTEBUF) */
-	auxmap__store_iovec_data_param_noinline(auxmap, msg_iov, msg_iovlen, snaplen);
+	auxmap__store_iovec_data_param_mmsg(auxmap, msg_iov, msg_iovlen, snaplen, use_bpf_loop);
 
 	/* Parameter 5: tuple (type: PT_SOCKTUPLE)*/
 	auxmap__store_socktuple_param_noinline(auxmap, data->fd, OUTBOUND, msg_name);
@@ -80,6 +80,12 @@ static __always_inline long handle_exit(uint32_t index, void *ctx) {
 	return 0;
 }
 
+/* bpf_loop callback used by sendmmsg_x. That program is loaded only when the
+ * bpf_loop helper is available, so the per-message iovec store uses bpf_loop. */
+static long handle_exit(uint32_t index, void *ctx) {
+	return sendmmsg_handle_msg(index, (sendmmsg_exit_t *)ctx, true);
+}
+
 SEC("tp_btf/sys_exit")
 int BPF_PROG(sendmmsg_x, struct pt_regs *regs, long ret) {
 	if(ret <= 0) {
@@ -168,8 +174,8 @@ int BPF_PROG(sendmmsg_old_x, struct pt_regs *regs, long ret) {
 	        .mmh = (struct mmsghdr *)args[1],
 	};
 
-	// Only first message
-	handle_exit(0, &data);
+	// Only first message; legacy program runs without bpf_loop.
+	sendmmsg_handle_msg(0, &data, false);
 
 	return 0;
 }