To send frames on the tap interface, the UDP uses a fairly complicated two
level batching. First multiple frames are gathered into a single "message"
for the qemu stream socket, then multiple messages are send with
sendmmsg(). We now have tap_send_frames() which already deals with sending
a number of frames, including batching and handling partial sends. Use
that to considerably simplify things.
This does make a couple of behavioural changes:
* We no longer split messages to keep them under 64kiB, which comments
say is necessary. However the TCP code didn't have equivalent code, so
either this isn't actually needed, or we should implement for both
(which is now easier since it can be done in one place).
* Previously when we got a partial send on UDP, we would resend the
remainder of the entire "message", including multiple frames. The
common code now only resends the remainder of a single frame, simply
dropping any frames which weren't even partially sent. This is what
TCP always did and is probably a better idea for UDP too.
Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
pcap.c | 27 -----------
pcap.h | 1 -
udp.c | 145 +++------------------------------------------------------
udp.h | 2 +-
4 files changed, 8 insertions(+), 167 deletions(-)
diff --git a/pcap.c b/pcap.c
index 8af9021..36f16b5 100644
--- a/pcap.c
+++ b/pcap.c
@@ -120,33 +120,6 @@ void pcap_multiple(const struct iovec *iov, unsigned int n, size_t
offset)
iov[i].iov_len - offset, &tv);
}
-/**
- * pcapm() - Capture multiple frames from multiple message headers to pcap file
- * @mmh: Pointer to first sendmmsg() header
- */
-void pcapmm(const struct mmsghdr *mmh, unsigned int vlen)
-{
- struct iovec *iov;
- struct timeval tv;
- unsigned int i, j;
-
- if (pcap_fd == -1)
- return;
-
- gettimeofday(&tv, NULL);
-
- for (i = 0; i < vlen; i++) {
- const struct msghdr *mh = &mmh[i].msg_hdr;
-
- for (j = 0; j < mh->msg_iovlen; j++) {
- iov = &mh->msg_iov[j];
-
- pcap_frame((char *)iov->iov_base + 4, iov->iov_len - 4,
- &tv);
- }
- }
-}
-
/**
* pcap_init() - Initialise pcap file
* @c: Execution context
diff --git a/pcap.h b/pcap.h
index eafc89b..c2af3cf 100644
--- a/pcap.h
+++ b/pcap.h
@@ -8,7 +8,6 @@
void pcap(const char *pkt, size_t len);
void pcap_multiple(const struct iovec *iov, unsigned int n, size_t offset);
-void pcapmm(const struct mmsghdr *mmh, unsigned int vlen);
void pcap_init(struct ctx *c);
#endif /* PCAP_H */
diff --git a/udp.c b/udp.c
index f7b9bdc..0c81283 100644
--- a/udp.c
+++ b/udp.c
@@ -228,9 +228,6 @@ static struct iovec udp6_l2_iov_tap [UDP_MAX_FRAMES];
static struct mmsghdr udp4_l2_mh_sock [UDP_MAX_FRAMES];
static struct mmsghdr udp6_l2_mh_sock [UDP_MAX_FRAMES];
-static struct mmsghdr udp4_l2_mh_tap [UDP_MAX_FRAMES];
-static struct mmsghdr udp6_l2_mh_tap [UDP_MAX_FRAMES];
-
/* recvmmsg()/sendmmsg() data for "spliced" connections */
static struct iovec udp4_iov_splice [UDP_MAX_FRAMES];
static struct iovec udp6_iov_splice [UDP_MAX_FRAMES];
@@ -347,16 +344,11 @@ static void udp_sock4_iov_init(const struct ctx *c)
mh->msg_iovlen = 1;
}
- for (i = 0, h = udp4_l2_mh_tap; i < UDP_MAX_FRAMES; i++, h++) {
- struct msghdr *mh = &h->msg_hdr;
-
+ for (i = 0; i < UDP_MAX_FRAMES; i++, h++) {
if (c->mode == MODE_PASTA)
udp4_l2_iov_tap[i].iov_base = &udp4_l2_buf[i].eh;
else
udp4_l2_iov_tap[i].iov_base = &udp4_l2_buf[i].vnet_len;
-
- mh->msg_iov = &udp4_l2_iov_tap[i];
- mh->msg_iovlen = 1;
}
}
@@ -392,16 +384,11 @@ static void udp_sock6_iov_init(const struct ctx *c)
mh->msg_iovlen = 1;
}
- for (i = 0, h = udp6_l2_mh_tap; i < UDP_MAX_FRAMES; i++, h++) {
- struct msghdr *mh = &h->msg_hdr;
-
+ for (i = 0; i < UDP_MAX_FRAMES; i++, h++) {
if (c->mode == MODE_PASTA)
udp6_l2_iov_tap[i].iov_base = &udp6_l2_buf[i].eh;
else
udp6_l2_iov_tap[i].iov_base = &udp6_l2_buf[i].vnet_len;
-
- mh->msg_iov = &udp6_l2_iov_tap[i];
- mh->msg_iovlen = 1;
}
}
@@ -741,102 +728,6 @@ static size_t udp_update_hdr6(const struct ctx *c, int n, in_port_t
dstport,
return buf_len;
}
-/**
- * udp_tap_send_pasta() - Send datagrams to the pasta tap interface
- * @c: Execution context
- * @mmh: Array of message headers to send
- * @n: Number of message headers to send
- *
- * #syscalls:pasta write
- */
-static void udp_tap_send_pasta(const struct ctx *c, struct mmsghdr *mmh,
- unsigned int n)
-{
- unsigned int i, j;
-
- for (i = 0; i < n; i++) {
- for (j = 0; j < mmh[i].msg_hdr.msg_iovlen; j++) {
- struct iovec *iov = &mmh[i].msg_hdr.msg_iov[j];
-
- /* We can't use writev() because the tap
- * character device relies on the write()
- * boundaries to discern frame boundaries
- */
- if (write(c->fd_tap, iov->iov_base, iov->iov_len) < 0)
- debug("tap write: %s", strerror(errno));
- else
- pcap(iov->iov_base, iov->iov_len);
- }
- }
-}
-
-/**
- * udp_tap_send_passt() - Send datagrams to the passt tap interface
- * @c: Execution context
- * @mmh: Array of message headers to send
- * @n: Number of message headers to send
- *
- * #syscalls:passt sendmmsg sendmsg
- */
-static void udp_tap_send_passt(const struct ctx *c, struct mmsghdr *mmh, int n)
-{
- struct msghdr *last_mh;
- ssize_t missing = 0;
- size_t msg_len = 0;
- unsigned int i;
- int ret;
-
- ret = sendmmsg(c->fd_tap, mmh, n, MSG_NOSIGNAL | MSG_DONTWAIT);
- if (ret <= 0)
- return;
-
- /* If we lose some messages to sendmmsg() here, fine, it's UDP. However,
- * the last message needs to be delivered completely, otherwise qemu
- * will fail to reassemble the next message and close the connection. Go
- * through headers from the last sent message, counting bytes, and, if
- * and as soon as we see more bytes than sendmmsg() sent, re-send the
- * rest with a blocking call.
- *
- * In pictures, given this example:
- *
- * iov #0 iov #1 iov #2 iov #3
- * tap_mmh[ret - 1].msg_hdr: .... ...... ..... ......
- * tap_mmh[ret - 1].msg_len: 7 .... ...
- *
- * when 'msglen' reaches: 10 ^
- * and 'missing' below is: 3 ---
- *
- * re-send everything from here: ^-- ----- ------
- */
- last_mh = &mmh[ret - 1].msg_hdr;
- for (i = 0; i < last_mh->msg_iovlen; i++) {
- if (missing <= 0) {
- msg_len += last_mh->msg_iov[i].iov_len;
- missing = msg_len - mmh[ret - 1].msg_len;
- }
-
- if (missing > 0) {
- uint8_t **iov_base;
- int first_offset;
-
- iov_base = (uint8_t **)&last_mh->msg_iov[i].iov_base;
- first_offset = last_mh->msg_iov[i].iov_len - missing;
- *iov_base += first_offset;
- last_mh->msg_iov[i].iov_len = missing;
-
- last_mh->msg_iov = &last_mh->msg_iov[i];
-
- if (sendmsg(c->fd_tap, last_mh, MSG_NOSIGNAL) < 0)
- debug("UDP: %li bytes to tap missing", missing);
-
- *iov_base -= first_offset;
- break;
- }
- }
-
- pcapmm(mmh, ret);
-}
-
/**
* udp_tap_send() - Prepare UDP datagrams and send to tap interface
* @c: Execution context
@@ -848,25 +739,18 @@ static void udp_tap_send_passt(const struct ctx *c, struct mmsghdr
*mmh, int n)
*
* Return: size of tap frame with headers
*/
-static void udp_tap_send(const struct ctx *c,
+static void udp_tap_send(struct ctx *c,
unsigned int start, unsigned int n,
in_port_t dstport, bool v6, const struct timespec *now)
{
- int msg_bufs = 0, msg_i = 0;
- struct mmsghdr *tap_mmh;
struct iovec *tap_iov;
- ssize_t msg_len = 0;
unsigned int i;
- if (v6) {
- tap_mmh = udp6_l2_mh_tap;
+ if (v6)
tap_iov = udp6_l2_iov_tap;
- } else {
- tap_mmh = udp4_l2_mh_tap;
+ else
tap_iov = udp4_l2_iov_tap;
- }
- tap_mmh[0].msg_hdr.msg_iov = &tap_iov[start];
for (i = start; i < start + n; i++) {
size_t buf_len;
@@ -876,24 +760,9 @@ static void udp_tap_send(const struct ctx *c,
buf_len = udp_update_hdr4(c, i, dstport, now);
tap_iov[i].iov_len = buf_len;
-
- /* With bigger messages, qemu closes the connection. */
- if (c->mode == MODE_PASST && msg_bufs &&
- msg_len + buf_len > SHRT_MAX) {
- tap_mmh[msg_i].msg_hdr.msg_iovlen = msg_bufs;
- msg_i++;
- tap_mmh[msg_i].msg_hdr.msg_iov = &tap_iov[i];
- msg_len = msg_bufs = 0;
- }
- msg_len += buf_len;
- msg_bufs++;
}
- tap_mmh[msg_i].msg_hdr.msg_iovlen = msg_bufs;
- if (c->mode == MODE_PASTA)
- udp_tap_send_pasta(c, tap_mmh, msg_i + 1);
- else
- udp_tap_send_passt(c, tap_mmh, msg_i + 1);
+ tap_send_frames(c, tap_iov + start, n);
}
/**
@@ -905,7 +774,7 @@ static void udp_tap_send(const struct ctx *c,
*
* #syscalls recvmmsg
*/
-void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events,
+void udp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events,
const struct timespec *now)
{
in_port_t dstport = ref.r.p.udp.udp.port;
diff --git a/udp.h b/udp.h
index 2a03335..68082ea 100644
--- a/udp.h
+++ b/udp.h
@@ -8,7 +8,7 @@
#define UDP_TIMER_INTERVAL 1000 /* ms */
-void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events,
+void udp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events,
const struct timespec *now);
int udp_tap_handler(struct ctx *c, int af, const void *addr,
const struct pool *p, const struct timespec *now);
--
2.38.1