udp_sock_handler() has a surprising difference in flow between pasta and passt mode: For pasta we send each frame to the tap interface as we prepare it. For passt, though, we prepare all the frames, then send them with a single sendmmsg(). Alter the pasta path to also prepare all the frames, then send them at the end. We already have a suitable data structure for the passt case. This will make it easier to abstract out the tap backend difference in future. Signed-off-by: David Gibson --- udp.c | 61 ++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/udp.c b/udp.c index 4610a02..a27fe88 100644 --- a/udp.c +++ b/udp.c @@ -783,6 +783,35 @@ static size_t udp_update_hdr6(const struct ctx *c, int n, in_port_t dstport, return buf_len; } +/** + * udp_tap_send_pasta() - Send datagrams to the pasta tap interface + * @c: Execution context + * @mmh: Array of message headers to send + * @n: Number of message headers to send + * + * #syscalls:pasta write + */ +static void udp_tap_send_pasta(const struct ctx *c, struct mmsghdr *mmh, + unsigned int n) +{ + unsigned int i, j; + + for (i = 0; i < n; i++) { + for (j = 0; j < mmh[i].msg_hdr.msg_iovlen; j++) { + struct iovec *iov = &mmh[i].msg_hdr.msg_iov[j]; + + /* We can't use writev() because the tap + * character device relies on the write() + * boundaries to discern frame boundaries + */ + if (write(c->fd_tap, iov->iov_base, iov->iov_len) < 0) + debug("tap write: %s", strerror(errno)); + else + pcap(iov->iov_base, iov->iov_len); + } + } +} + /** * udp_sock_handler() - Handle new data from socket * @c: Execution context @@ -835,31 +864,25 @@ void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, else buf_len = udp_update_hdr4(c, i, dstport, now); - if (c->mode == MODE_PASTA) { - void *frame = tap_iov[i].iov_base; + tap_iov[i].iov_len = buf_len; - if (write(c->fd_tap, frame, buf_len) < 0) - debug("tap write: %s", strerror(errno)); - pcap(frame, buf_len); - } else { - tap_iov[i].iov_len = buf_len; - - /* With bigger messages, qemu closes the connection. */ - if (msg_bufs && msg_len + buf_len > SHRT_MAX) { - tap_mmh[msg_i].msg_hdr.msg_iovlen = msg_bufs; - msg_i++; - tap_mmh[msg_i].msg_hdr.msg_iov = &tap_iov[i]; - msg_len = msg_bufs = 0; - } - - msg_len += buf_len; - msg_bufs++; + /* With bigger messages, qemu closes the connection. */ + if (c->mode == MODE_PASST && msg_bufs && + msg_len + buf_len > SHRT_MAX) { + tap_mmh[msg_i].msg_hdr.msg_iovlen = msg_bufs; + msg_i++; + tap_mmh[msg_i].msg_hdr.msg_iov = &tap_iov[i]; + msg_len = msg_bufs = 0; } + msg_len += buf_len; + msg_bufs++; } tap_mmh[msg_i].msg_hdr.msg_iovlen = msg_bufs; - if (c->mode == MODE_PASTA) + if (c->mode == MODE_PASTA) { + udp_tap_send_pasta(c, tap_mmh, msg_i + 1); return; + } ret = sendmmsg(c->fd_tap, tap_mmh, msg_i + 1, MSG_NOSIGNAL | MSG_DONTWAIT); -- 2.38.1

The receive part of udp_sock_handler() and udp_sock_handler_splice() is now almost identical. In preparation for merging that, split the receive part of udp_sock_handler() from the part preparing and sending the frames for sending on the tap interface. The latter goes into a new udp_tap_send() function. Signed-off-by: David Gibson --- udp.c | 79 +++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/udp.c b/udp.c index 7281bc3..64c9219 100644 --- a/udp.c +++ b/udp.c @@ -880,51 +880,39 @@ static void udp_tap_send_passt(const struct ctx *c, struct mmsghdr *mmh, int n) } /** - * udp_sock_handler() - Handle new data from socket + * udp_tap_send() - Prepare UDP datagrams and send to tap interface * @c: Execution context - * @ref: epoll reference - * @events: epoll events bitmap + * @start: Index of first datagram in udp[46]_l2_buf pool + * @n: Number of datagrams to send + * @dstport: Destination port number + * @v6: True if using IPv6 * @now: Current timestamp * - * #syscalls recvmmsg + * Return: size of tap frame with headers */ -void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, - const struct timespec *now) +static void udp_tap_send(const struct ctx *c, + unsigned int start, unsigned int n, + in_port_t dstport, bool v6, const struct timespec *now) { - in_port_t dstport = ref.r.p.udp.udp.port; - struct mmsghdr *tap_mmh, *sock_mmh; int msg_bufs = 0, msg_i = 0; - ssize_t n, msg_len = 0; + struct mmsghdr *tap_mmh; struct iovec *tap_iov; + ssize_t msg_len = 0; unsigned int i; - if (events == EPOLLERR) - return; - - if (ref.r.p.udp.udp.splice) { - udp_sock_handler_splice(c, ref, events, now); - return; - } - - if (ref.r.p.udp.udp.v6) { + if (v6) { tap_mmh = udp6_l2_mh_tap; - sock_mmh = udp6_l2_mh_sock; tap_iov = udp6_l2_iov_tap; } else { tap_mmh = udp4_l2_mh_tap; - sock_mmh = udp4_l2_mh_sock; tap_iov = udp4_l2_iov_tap; } - n = recvmmsg(ref.r.s, sock_mmh, UDP_TAP_FRAMES, 0, NULL); - if (n <= 0) - return; - - tap_mmh[0].msg_hdr.msg_iov = &tap_iov[0]; - for (i = 0; i < (unsigned)n; i++) { + tap_mmh[0].msg_hdr.msg_iov = &tap_iov[start]; + for (i = start; i < start + n; i++) { size_t buf_len; - if (ref.r.p.udp.udp.v6) + if (v6) buf_len = udp_update_hdr6(c, i, dstport, now); else buf_len = udp_update_hdr4(c, i, dstport, now); @@ -950,6 +938,43 @@ void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, udp_tap_send_passt(c, tap_mmh, msg_i + 1); } +/** + * udp_sock_handler() - Handle new data from socket + * @c: Execution context + * @ref: epoll reference + * @events: epoll events bitmap + * @now: Current timestamp + * + * #syscalls recvmmsg + */ +void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, + const struct timespec *now) +{ + in_port_t dstport = ref.r.p.udp.udp.port; + bool v6 = ref.r.p.udp.udp.v6; + struct mmsghdr *sock_mmh; + ssize_t n; + + if (events == EPOLLERR) + return; + + if (ref.r.p.udp.udp.splice) { + udp_sock_handler_splice(c, ref, events, now); + return; + } + + if (ref.r.p.udp.udp.v6) + sock_mmh = udp6_l2_mh_sock; + else + sock_mmh = udp4_l2_mh_sock; + + n = recvmmsg(ref.r.s, sock_mmh, UDP_TAP_FRAMES, 0, NULL); + if (n <= 0) + return; + + udp_tap_send(c, 0, n, dstport, v6, now); +} + /** * udp_tap_handler() - Handle packets from tap * @c: Execution context -- 2.38.1

udp_splice_namebuf is now used only for spliced sending, and so it is only ever populated with the localhost address, either IPv4 or IPv6. So, replace the awkward initialization in udp_sock_handler_splice() with statically initialized versions for IPv4 and IPv6. We then just need to update the port number in udp_sock_handler_splice(). Signed-off-by: David Gibson --- udp.c | 40 ++++++++++++++++++---------------------- util.h | 7 +++++++ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/udp.c b/udp.c index 24fa984..98d8687 100644 --- a/udp.c +++ b/udp.c @@ -232,11 +232,18 @@ static struct mmsghdr udp4_l2_mh_tap [UDP_MAX_FRAMES]; static struct mmsghdr udp6_l2_mh_tap [UDP_MAX_FRAMES]; /* recvmmsg()/sendmmsg() data for "spliced" connections */ -static struct sockaddr_storage udp_splice_namebuf; - static struct iovec udp4_iov_splice [UDP_MAX_FRAMES]; static struct iovec udp6_iov_splice [UDP_MAX_FRAMES]; +static struct sockaddr_in udp4_localname = { + .sin_family = AF_INET, + .sin_addr = IN4ADDR_LOOPBACK_INIT, +}; +static struct sockaddr_in6 udp6_localname = { + .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT, +}; + static struct mmsghdr udp4_mh_splice [UDP_MAX_FRAMES]; static struct mmsghdr udp6_mh_splice [UDP_MAX_FRAMES]; @@ -610,23 +617,10 @@ static void udp_sock_handler_splice(const struct ctx *c, union epoll_ref ref, if (n <= 0) return; - if (v6) { - *((struct sockaddr_in6 *)&udp_splice_namebuf) = - ((struct sockaddr_in6) { - .sin6_family = AF_INET6, - .sin6_addr = IN6ADDR_LOOPBACK_INIT, - .sin6_port = htons(dst), - .sin6_scope_id = 0, - }); - } else { - *((struct sockaddr_in *)&udp_splice_namebuf) = - ((struct sockaddr_in) { - .sin_family = AF_INET, - .sin_addr = { .s_addr = htonl(INADDR_LOOPBACK) }, - .sin_port = htons(dst), - .sin_zero = { 0 }, - }); - } + if (v6) + udp6_localname.sin6_port = htons(dst); + else + udp4_localname.sin_port = htons(dst); for (i = 0; i < n; i += m) { in_port_t src = sa_port(v6, mmh_recv[i].msg_hdr.msg_name); @@ -1252,9 +1246,11 @@ static void udp_splice_iov_init(void) struct msghdr *mh4 = &udp4_mh_splice[i].msg_hdr; struct msghdr *mh6 = &udp6_mh_splice[i].msg_hdr; - mh4->msg_name = mh6->msg_name = &udp_splice_namebuf; - mh4->msg_namelen = sizeof(udp_splice_namebuf); - mh6->msg_namelen = sizeof(udp_splice_namebuf); + mh4->msg_name = &udp4_localname; + mh4->msg_namelen = sizeof(udp4_localname); + + mh6->msg_name = &udp6_localname; + mh6->msg_namelen = sizeof(udp6_localname); udp4_iov_splice[i].iov_base = udp4_l2_buf[i].data; udp6_iov_splice[i].iov_base = udp6_l2_buf[i].data; diff --git a/util.h b/util.h index 02b55a9..3baec91 100644 --- a/util.h +++ b/util.h @@ -79,6 +79,13 @@ (IN_MULTICAST(ntohl((a)->s_addr))) #define IN4_ARE_ADDR_EQUAL(a, b) \ (((struct in_addr *)(a))->s_addr == ((struct in_addr *)b)->s_addr) +#if __BYTE_ORDER == __BIG_ENDIAN +#define IN4ADDR_LOOPBACK_INIT \ + { .s_addr = INADDR_LOOPBACK } +#else +#define IN4ADDR_LOOPBACK_INIT \ + { .s_addr = __bswap_constant_32(INADDR_LOOPBACK) } +#endif #define NS_FN_STACK_SIZE (RLIMIT_STACK_VAL * 1024 / 8) int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, -- 2.38.1

Currently we have special sockets for receiving datagrams from locahost which can use the optimized "splice" path rather than going across the tap interface. We want to loosen this so that sockets can receive sockets that will be forwarded by both the spliced and non-spliced paths. To do this, we alter the meaning of the @splice bit in the reference to mean that packets receieved on this socket *can* be spliced, not that they *will* be spliced. They'll only actually be spliced if they come from 127.0.0.1 or ::1. We can't (for now) remove the splice bit entirely, unlike with TCP. Our gateway mapping means that if the ns initiates communication to the gw address, we'll translate that to target 127.0.0.1 on the host side. Reply packets will therefore have source address 127.0.0.1 when received on the host, but these need to go via the tap path where that will be translated back to the gateway address. We need the @splice bit to distinguish that case from packets going from localhost to a port mapped explicitly with -u which should be spliced. Signed-off-by: David Gibson --- udp.c | 52 +++++++++++++++++++++++++++++++++------------------- udp.h | 2 +- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/udp.c b/udp.c index ed31216..8ba3453 100644 --- a/udp.c +++ b/udp.c @@ -513,16 +513,25 @@ static int udp_splice_new_ns(void *arg) } /** - * sa_port() - Determine port from a sockaddr_in or sockaddr_in6 + * udp_mmh_splice_port() - Is source address of message suitable for splicing? * @v6: Is @sa a sockaddr_in6 (otherwise sockaddr_in)? - * @sa: Pointer to either sockaddr_in or sockaddr_in6 + * @mmh: mmsghdr of incoming message + * + * Return: if @sa refers to localhost (127.0.0.1 or ::1) the port from + * @sa in host order, otherwise -1. */ -static in_port_t sa_port(bool v6, const void *sa) +static int udp_mmh_splice_port(bool v6, const struct mmsghdr *mmh) { - const struct sockaddr_in6 *sa6 = sa; - const struct sockaddr_in *sa4 = sa; + const struct sockaddr_in6 *sa6 = mmh->msg_hdr.msg_name; + const struct sockaddr_in *sa4 = mmh->msg_hdr.msg_name; + + if (v6 && IN6_IS_ADDR_LOOPBACK(&sa6->sin6_addr)) + return ntohs(sa6->sin6_port); + + if (!v6 && IN4_IS_ADDR_LOOPBACK(&sa4->sin_addr)) + return ntohs(sa4->sin_port); - return v6 ? ntohs(sa6->sin6_port) : ntohs(sa4->sin_port); + return -1; } /** @@ -918,23 +927,28 @@ void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, if (n <= 0) return; - if (!ref.r.p.udp.udp.splice) { - udp_tap_send(c, 0, n, dstport, v6, now); - return; - } - for (i = 0; i < n; i += m) { - in_port_t src = sa_port(v6, mmh_recv[i].msg_hdr.msg_name); + int splicefrom = -1; + m = n; + + if (ref.r.p.udp.udp.splice) { + splicefrom = udp_mmh_splice_port(v6, mmh_recv + i); + + for (m = 1; i + m < n; m++) { + int p; - for (m = 1; i + m < n; m++) { - void *mname = mmh_recv[i + m].msg_hdr.msg_name; - if (sa_port(v6, mname) != src) - break; + p = udp_mmh_splice_port(v6, mmh_recv + i + m); + if (p != splicefrom) + break; + } } - udp_splice_sendfrom(c, i, m, src, dstport, v6, - ref.r.p.udp.udp.ns, ref.r.p.udp.udp.orig, - now); + if (splicefrom >= 0) + udp_splice_sendfrom(c, i, m, splicefrom, dstport, + v6, ref.r.p.udp.udp.ns, + ref.r.p.udp.udp.orig, now); + else + udp_tap_send(c, i, m, dstport, v6, now); } } diff --git a/udp.h b/udp.h index 053991e..2a03335 100644 --- a/udp.h +++ b/udp.h @@ -22,7 +22,7 @@ void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s, /** * union udp_epoll_ref - epoll reference portion for TCP connections * @bound: Set if this file descriptor is a bound socket - * @splice: Set if descriptor is associated to "spliced" connection + * @splice: Set if descriptor packets to be "spliced" * @orig: Set if a spliced socket which can originate "connections" * @ns: Set if this is a socket in the pasta network namespace * @v6: Set for IPv6 sockets or connections -- 2.38.1