Add vhost.c and vhost.h which define the api and implementation for pasta vhost acceleration - vq_state/vhost_vq_state[]: our own view of each virtqueue, which the kernel doesn't track for us: descriptors ready to hand back, how far we've read the used ring, and the next descriptor to allocate - vring_desc, vring_avail_all, vring_used_all, vhost_memory: the rings and the region table shared with the kernel - vhost_setup_net: open /dev/vhost-net and negotiate VIRTIO_F_VERSION_1 and VHOST_NET_F_VIRTIO_NET_HDR. Failure is fatal only if --vhost-kernel is set to 'on'. - vhost_setup_eventfds: create, register and watch the call, kick and error eventfds for one queue - vhost_setup_memory_table: register the buffers the kernel may touch. pasta has no guest, so each region maps guest_phys_addr onto the identical userspace_addr, making GPA translation an identity mapping. This function uses the *_register_memory_regions functions created per protocol and for general purpose buffers. - vhost_set_vring: publish a vring's addresses, bind the tap fd as its backend and lay out its descriptors - vhost_rx_descriptor_handoff, vhost_kick: hand consumed descriptors back and notify the kernel unless it says it's already polling Signed-off-by: Eugenio Pérez Signed-off-by: Ammar Yasser --- Makefile | 2 +- tcp_buf.c | 22 +++- tcp_buf.h | 4 + udp.c | 20 ++++ udp.h | 3 + util.c | 18 ++++ util.h | 2 + vhost.c | 295 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++++++ 9 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 vhost.c create mode 100644 vhost.h diff --git a/Makefile b/Makefile index f6c87b1..6f11e4b 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ PASST_SRCS = arch.c arp.c bitmap.c checksum.c conf.c dhcp.c dhcpv6.c \ isolation.c lineread.c log.c mld.c ndp.c netlink.c migrate.c packet.c \ parse.c passt.c pasta.c pcap.c pif.c repair.c serialise.c tap.c tcp.c \ tcp_buf.c tcp_splice.c tcp_vu.c udp.c udp_flow.c udp_vu.c util.c \ - vhost_user.c virtio.c vu_common.c + vhost.c vhost_user.c virtio.c vu_common.c PASST_REPAIR_SRCS = passt-repair.c PESTO_SRCS = pesto.c bitmap.c fwd_rule.c inany.c ip.c lineread.c parse.c \ serialise.c diff --git a/tcp_buf.c b/tcp_buf.c index 72c4541..dd80384 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -31,12 +31,14 @@ #include "inany.h" #include "tcp_conn.h" #include "tcp_internal.h" -#include "tcp_buf.h" +#include "vhost.h" #define TCP_FRAMES_MEM 128 #define TCP_FRAMES \ (c->mode == MODE_PASTA ? 1 : TCP_FRAMES_MEM) +#define TCP_NREGIONS 5 + /* Static buffers */ /* Ethernet header for IPv4 and IPv6 frames */ @@ -75,6 +77,24 @@ void tcp_update_l2_buf(const unsigned char *eth_d) eth_update_mac(&tcp_eth_hdr[i], eth_d, NULL); } +/** + * tcp_register_memory_region() - Register the TCP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + TCP_NREGIONS) > N_VHOST_REGIONS) + die("tcp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + TCP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload_tap_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp4_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp6_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_eth_hdr); +} + /** * tcp_sock_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets * @c: Execution context diff --git a/tcp_buf.h b/tcp_buf.h index 5d31cea..5e341ed 100644 --- a/tcp_buf.h +++ b/tcp_buf.h @@ -6,11 +6,15 @@ #ifndef TCP_BUF_H #define TCP_BUF_H +union vhost_memory_u; +struct tcp_tap_conn; + void tcp_sock_iov_init(const struct ctx *c); void tcp_payload_flush(const struct ctx *c, const struct timespec *now); int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, uint32_t already_sent, const struct timespec *now); int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, const struct timespec *now); +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); #endif /*TCP_BUF_H */ diff --git a/udp.c b/udp.c index 505e554..a4355a1 100644 --- a/udp.c +++ b/udp.c @@ -118,6 +118,7 @@ #include "udp_internal.h" #include "udp_vu.h" #include "epoll_ctl.h" +#include "vhost.h" #define UDP_MAX_FRAMES 32 /* max # of frames to receive at once */ @@ -128,6 +129,8 @@ #define UDP_TIMEOUT_DEFAULT 30 /* s */ #define UDP_TIMEOUT_STREAM_DEFAULT 120 /* s */ +#define UDP_NREGIONS 5 + /* Maximum UDP data to be returned in ICMP messages */ #define ICMP4_MAX_DLEN 8 #define ICMP6_MAX_DLEN (IPV6_MIN_MTU \ @@ -448,6 +451,23 @@ static void udp_send_tap_icmp4(const struct ctx *c, tap_icmp4_send(c, saddr, eaddr, &msg, tap_omac, msglen); } +/** + * udp_register_memory_region() - Register the UDP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + UDP_NREGIONS) > N_VHOST_REGIONS) + die("udp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + UDP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_eth_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_iov_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_mh_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_meta); +} /** * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer diff --git a/udp.h b/udp.h index b50283e..ebf1a4e 100644 --- a/udp.h +++ b/udp.h @@ -11,6 +11,8 @@ #include "fwd.h" +union vhost_memory_u; + void udp_listen_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, const struct timespec *now); void udp_sock_handler(const struct ctx *c, union epoll_ref ref, @@ -21,6 +23,7 @@ int udp_tap_handler(const struct ctx *c, uint8_t pif, const struct timespec *now); int udp_init(struct ctx *c); void udp_update_l2_buf(const unsigned char *eth_d); +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * struct udp_ctx - Execution context for UDP diff --git a/util.c b/util.c index ff1f19f..cafe0ec 100644 --- a/util.c +++ b/util.c @@ -33,10 +33,13 @@ #include "log.h" #include "pcap.h" #include "serialise.h" +#include "vhost.h" #ifdef HAS_GETRANDOM #include #endif +#define GENERAL_NREGIONS 2 + /* Zero-filled buffer to pad 802.3 frames, up to 60 (ETH_ZLEN) bytes */ uint8_t eth_pad[ETH_ZLEN] = { 0 }; @@ -453,6 +456,21 @@ int open_in_ns(const struct ctx *c, const char *path, int flags) return arg.fd; } +/** + * general_register_memory_region() - Register the general (non protocol specific) buffers + * into a memory struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + GENERAL_NREGIONS) > N_VHOST_REGIONS) + die("general: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + GENERAL_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(pkt_buf); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(eth_pad); +} + /** * pidfile_write() - Write PID to file, if requested to do so, and close it * @fd: Open PID file descriptor, closed on exit, -1 to skip writing it diff --git a/util.h b/util.h index ec2d9ac..11f6877 100644 --- a/util.h +++ b/util.h @@ -145,6 +145,7 @@ int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, struct ctx; union sockaddr_inany; +union vhost_memory_u; int sock_l4(const struct ctx *c, enum epoll_type type, const union sockaddr_inany *sa, const char *ifname); @@ -170,6 +171,7 @@ int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, int read_remainder(int fd, const struct iovec *iov, size_t cnt, size_t skip); bool snprintf_check(char *str, size_t size, const char *format, ...); long clamped_scale(long x, long y, long lo, long hi, long f); +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * af_name() - Return name of an address family diff --git a/vhost.c b/vhost.c new file mode 100644 index 0000000..a69c4b5 --- /dev/null +++ b/vhost.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Ammar Yasser + * + * vhost.c - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "passt.h" +#include "vhost.h" +#include "epoll_ctl.h" +#include "udp.h" +#include "tcp_buf.h" + +struct vq_state vhost_vq_state[2]; + +struct vring_desc_pasta vring_desc[2][VHOST_NDESCS] + __attribute__((aligned(PAGE_SIZE))); + +struct vring_avail_pasta vring_avail_all[2] + __attribute__((aligned(PAGE_SIZE))); +struct vring_used_pasta vring_used_all[2] + __attribute__((aligned(PAGE_SIZE))); + +union vhost_memory_u vhost_memory = { + .mem = { + .nregions = N_VHOST_REGIONS, + }, +}; + +/** + * vhost_setup_net() - Open and negotiate features on /dev/vhost-net + * @c: Execution context; c->vhost.fd and c->vhost.features are set + * on success, and left untouched on failure + * + * Failure here means vhost-net isn't usable, not that pasta can't run: the + * caller falls back to plain tap operation unless acceleration was required. + * That's why nothing in here is fatal. + * + * Return: 0 on success, -1 if vhost-net is unavailable or unusable + */ +int vhost_setup_net(struct ctx *c) +{ + const uint64_t req_features = (1ULL << VIRTIO_F_VERSION_1) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t features; + int vhost_fd; + + vhost_fd = open("/dev/vhost-net", O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (vhost_fd < 0) { + debug_perror("Couldn't open /dev/vhost-net"); + return -1; + } + + if (ioctl(vhost_fd, VHOST_SET_OWNER, NULL) < 0) { + debug_perror("VHOST_SET_OWNER ioctl failed"); + goto close_fd; + } + + if (ioctl(vhost_fd, VHOST_GET_FEATURES, &features) < 0) { + debug_perror("VHOST_GET_FEATURES ioctl failed"); + goto close_fd; + } + + if ((features & req_features) != req_features) { + debug("vhost-net is missing features: 0x%016" PRIx64, + req_features & ~features); + goto close_fd; + } + + features = req_features; + if (ioctl(vhost_fd, VHOST_SET_FEATURES, &features) < 0) { + debug_perror("VHOST_SET_FEATURES ioctl failed"); + goto close_fd; + } + + c->vhost.features = features; + c->vhost.fd = vhost_fd; + + return 0; + +close_fd: + close(vhost_fd); + + return -1; +} + +/** + * vhost_setup_eventfds() - Set up one queue's eventfds and ring size + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * + */ +void vhost_setup_eventfds(struct ctx *c, int queue_idx) +{ + struct vhost_vring_file vring_file = { .index = queue_idx }; + struct vhost_vring_state state = { .index = queue_idx }; + union epoll_ref ref = { 0 }; + int vhost_fd = c->vhost.fd; + int rc; + + state.num = VHOST_NDESCS; + ref.queue = queue_idx; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost call eventfd, queue %d", + queue_idx); + ref.fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_CALL, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_CALL ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_CALL; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost call eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].call_fd = vring_file.fd; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost error eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ERR, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_ERR ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_ERROR; + ref.fd = vring_file.fd; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost error eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].err_fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_NUM, &state); + if (rc < 0) + die_perror("VHOST_SET_VRING_NUM ioctl failed, queue %d", + queue_idx); + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost kick eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_KICK, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_KICK ioctl failed, queue %d", + queue_idx); + + c->vhost.vq[queue_idx].kick_fd = vring_file.fd; + + vhost_vq_state[queue_idx].num_free = VHOST_NDESCS; +} + +/** + * vhost_setup_memory_table() - Register the GPA/HVA translation table + * @c: Execution context; c->vhost.fd must already be set + * + * vhost-net reads the addresses we put in descriptors as guest physical + * addresses, and translates them through this table before touching the + * memory they refer to. pasta has no guest and no second address space: + * the addresses we put there are our own virtual addresses. Every region + * below therefore sets guest_phys_addr equal to userspace_addr, so that + * GPA == HVA and the translation is an identity mapping. + * + * Return: 0 on success, -1 on error with errno set + */ +int vhost_setup_memory_table(struct ctx *c) { + size_t region_idx = 0; + + /* general purpose buffers */ + general_register_memory_regions(&vhost_memory, ®ion_idx); + + /* tcp specific buffers */ + tcp_register_memory_regions(&vhost_memory, ®ion_idx); + + /* udp specific buffers */ + udp_register_memory_regions(&vhost_memory, ®ion_idx); + + vhost_memory.mem.nregions = region_idx; + + return ioctl(c->vhost.fd, VHOST_SET_MEM_TABLE, &vhost_memory.mem); +} + +/** + * vhost_set_vring() - Register a vring's addresses and bind its backend + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * @tap_fd: Tap fd to bind as this queue's backend + */ +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd) +{ + int vhost_fd = c->vhost.fd; + struct vhost_vring_addr addr = { + .index = queue_idx, + .desc_user_addr = (unsigned long)vring_desc[queue_idx], + .avail_user_addr = (unsigned long)&vring_avail_all[queue_idx], + .used_user_addr = (unsigned long)&vring_used_all[queue_idx], + .log_guest_addr = (unsigned long)&vring_used_all[queue_idx], + }; + struct vhost_vring_file file = { + .index = queue_idx, + .fd = tap_fd, + }; + unsigned int i; + int rc; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ADDR, &addr); + if (rc < 0) + die_perror("VHOST_SET_VRING_ADDR ioctl failed, queue %d", + queue_idx); + + if (queue_idx == VHOST_AVAIL_Q_IDX) { + for (i = 0; i < VHOST_NDESCS; ++i) { + vring_desc[VHOST_AVAIL_Q_IDX][i].addr = + (uintptr_t)pkt_buf + i * VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].len = VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].flags = + VRING_DESC_F_WRITE; + } + + for (i = 0; i < VHOST_NDESCS; ++i) + vring_avail_all[VHOST_AVAIL_Q_IDX].ring[i] = + htole16(i); + + vhost_rx_descriptor_handoff(c); + } + + if (queue_idx == VHOST_USED_Q_IDX) { + for (i = 0; i < (VHOST_NDESCS - 1); ++i) + vring_desc[VHOST_USED_Q_IDX][i].next = i + 1; + } + + rc = ioctl(vhost_fd, VHOST_NET_SET_BACKEND, &file); + if (rc < 0) + die_perror("VHOST_NET_SET_BACKEND ioctl failed, queue %d", + queue_idx); +} + +/** + * vhost_rx_descriptor_handoff() - Announce freed from-guest descriptors + * @c: Execution context + * + * Bumps avail.idx by the number of descriptors accumulated in + * vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free (from prior + * consume_one_rx_descriptor() calls), + * then resets the counter to zero. The kernel will see the new + * avail.idx and consume the freshly-available descriptors. + */ +void vhost_rx_descriptor_handoff(struct ctx *c) +{ + smp_wmb(); + + if (!vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free) + return; + + vring_avail_all[VHOST_AVAIL_Q_IDX].idx += + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free = 0; + vhost_kick(&vring_used_all[VHOST_AVAIL_Q_IDX], + c->vhost.vq[VHOST_AVAIL_Q_IDX].kick_fd); +} + +/** + * vhost_kick() - Notify the kernel that new descriptors are available + * @used: Used ring of the queue we're announcing on, checked to see + * whether the kernel's virtio thread is already reading + * descriptors and doesn't want to be notified + * @kick_fd: Kick eventfd of that same queue + */ +void vhost_kick(struct vring_used_pasta *used, int kick_fd) +{ + /* Ensure that the read of used->flags doesn't get reordered to be + * above the avail.idx update + */ + smp_mb(); + + if (!(used->flags & VRING_USED_F_NO_NOTIFY)) + eventfd_write(kick_fd, 1); +} diff --git a/vhost.h b/vhost.h new file mode 100644 index 0000000..bd555f9 --- /dev/null +++ b/vhost.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: Ammar Yasser + * + * vhost.h - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#ifndef VHOST_H +#define VHOST_H + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "passt.h" + +#define VHOST_AVAIL_Q_IDX 0 +#define VHOST_USED_Q_IDX 1 + +/** + * struct vq_state - Per-virtqueue local descriptor tracking + * @num_free: Number of descriptors ready to be announced to the + * kernel via vhost_rx_descriptor_handoff() + * @last_used_idx: Number of used-ring entries consumed so far; + * lagging read cursor vs. vring_used->idx (the + * kernel's write cursor) + */ +extern struct vq_state { + uint16_t num_free; + uint16_t last_used_idx; + uint16_t next_free; +} vhost_vq_state[2]; + +struct vring_desc_pasta { + __virtio64 addr; + __virtio32 len; + __virtio16 flags; + __virtio16 next; +}; + +extern struct vring_desc_pasta vring_desc[2][VHOST_NDESCS]; + +struct vring_avail_pasta { + __virtio16 flags; + __virtio16 idx; + __virtio16 ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_avail_pasta) == + offsetof(struct vring_avail, ring[VHOST_NDESCS]), + "vring_avail_pasta doesn't match the kernel avail ring layout"); +extern struct vring_avail_pasta vring_avail_all[2]; + +struct vring_used_pasta { + __virtio16 flags; + __virtio16 idx; + vring_used_elem_t ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_used_pasta) == + offsetof(struct vring_used, ring[VHOST_NDESCS]), + "vring_used_pasta doesn't match the kernel used ring layout"); +extern struct vring_used_pasta vring_used_all[2]; + +#define N_VHOST_REGIONS 12 +union vhost_memory_u { + struct vhost_memory mem; + char buf[offsetof(struct vhost_memory, regions[N_VHOST_REGIONS])]; +}; +extern union vhost_memory_u vhost_memory; + +#define VHOST_MEMORY_REGION_PTR(addr, size) \ + (struct vhost_memory_region) { \ + .guest_phys_addr = (uintptr_t)addr, \ + .memory_size = size, \ + .userspace_addr = (uintptr_t)addr, \ + } +#define VHOST_MEMORY_REGION(buf) VHOST_MEMORY_REGION_PTR(&buf, sizeof(buf)) + +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd); +int vhost_setup_memory_table(struct ctx *c); +int vhost_setup_net(struct ctx *c); +void vhost_setup_eventfds(struct ctx *c, int queue_idx); +void vhost_rx_descriptor_handoff(struct ctx *c); +void vhost_kick(struct vring_used_pasta *used, int kick_fd); + +#endif /* VHOST_H */ -- 2.39.5 (Apple Git-154)