October 2025 - dev

[PATCH v15 2/9] fwd: Add cache table for ARP/NDP contents
by Jon Maloy 24 Oct '25

24 Oct '25

We add a cache table to keep track of the contents of the kernel ARP and NDP tables. The table is fed from the just introduced netlink based neigbour subscription function. Signed-off-by: Jon Maloy <jmaloy(a)redhat.com> --- v5: - Moved to earlier in series to reduce rebase conflicts v6: - Sqashed the hash list commit and the FIFO/LRU queue commit - Removed hash lookup. We now only use linear lookup in a linked list - Eliminated dynamic memory allocation. - Ensured there is only one call to clock_gettime() - Using MAC_ZERO instead of the previously dedicated definitions v7: - NOW using MAC_ZERO where needed - I am still using linear back-off for empty cache entries. Even an incoming, flow-creating packet from a local host gives no guarantee that its MAC address is in the ARP table, so we must allow for a few new attempts at first possible occasions. Only after several failed lookups can we conclude that we probably never will succeed. Hence the back-off. - Fixed a bug that David inadvertently made me aware of: I only intended to set the initial expiry value to MAC_CACHE_RENEWAL when an ARP/NDP table lookup was successful. - Improved struct and function description comments. v8: - Total re-design of table, adapting to the new, subscription based way of updating it. v9: - Catering for MAC address change for an existing host. v10: - Changes according to feedback from David Gibson v12: - Changes according to feedback from David and Stefano - Added dummy entries for loopback and default GW addresses v13: - Changes according to feedback and discussions with David and Stefano v14: - Moved the call to nat_inbound() to a much more sensible place in netlink.c, as suggested by David. v15: - Changes according to feedback from David and Stefano - Removed 'blocker' entry for guest_gw. - Let blocker entry for ctx->map_guest_addr use our_mac_addr - We let fwd_neigh_mac_get() do a secondary lookup in the table, using the guest_gw address, when the primary lookup fails. This way, we always return the correct gateway MAC address if there happens to be a real host presenting that address. --- fwd.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ fwd.h | 7 ++ netlink.c | 12 +++- passt.c | 1 + 4 files changed, 220 insertions(+), 2 deletions(-) diff --git a/fwd.c b/fwd.c index 250cf56..93d83e3 100644 --- a/fwd.c +++ b/fwd.c @@ -26,6 +26,7 @@ #include "passt.h" #include "lineread.h" #include "flow_table.h" +#include "netlink.h" /* Empheral port range: values from RFC 6335 */ static in_port_t fwd_ephemeral_min = (1 << 15) + (1 << 14); @@ -33,6 +34,207 @@ static in_port_t fwd_ephemeral_max = NUM_PORTS - 1; #define PORT_RANGE_SYSCTL "/proc/sys/net/ipv4/ip_local_port_range" +#define NEIGH_TABLE_SLOTS 1024 +#define NEIGH_TABLE_SIZE (NEIGH_TABLE_SLOTS / 2) +static_assert((NEIGH_TABLE_SLOTS & (NEIGH_TABLE_SLOTS - 1)) == 0, + "NEIGH_TABLE_SLOTS must be a power of two"); + +/** + * struct neigh_table_entry - Entry in the ARP/NDP table + * @next: Next entry in slot or free list + * @addr: IP address of represented host + * @mac: MAC address of represented host + * @permanent: Entry cannot be altered or freed by notification + */ +struct neigh_table_entry { + struct neigh_table_entry *next; + union inany_addr addr; + uint8_t mac[ETH_ALEN]; + bool permanent; +}; + +/** + * struct neigh_table - Cache of ARP/NDP table contents + * @entries: Entries to be plugged into the hash slots when allocated + * @slots: Hash table slots + * @free: Linked list of unused entries + */ +struct neigh_table { + struct neigh_table_entry entries[NEIGH_TABLE_SIZE]; + struct neigh_table_entry *slots[NEIGH_TABLE_SLOTS]; + struct neigh_table_entry *free; +}; + +static struct neigh_table neigh_table; + +/** + * neigh_table_slot() - Hash key to a number within the table range + * @c: Execution context + * @key: The key to be used for the hash + * + * Return: the resulting hash value + */ +static size_t neigh_table_slot(const struct ctx *c, + const union inany_addr *key) +{ + struct siphash_state st = SIPHASH_INIT(c->hash_secret); + uint32_t i; + + inany_siphash_feed(&st, key); + i = siphash_final(&st, sizeof(*key), 0); + + return ((size_t)i) & (NEIGH_TABLE_SIZE - 1); +} + +/** + * fwd_neigh_table_find() - Find a MAC table entry + * @c: Execution context + * @addr: Neighbour address to be used as key for the lookup + * + * Return: the matching entry, if found. Otherwise NULL + */ +static struct neigh_table_entry *fwd_neigh_table_find(const struct ctx *c, + const union inany_addr *addr) +{ + size_t slot = neigh_table_slot(c, addr); + struct neigh_table_entry *e = neigh_table.slots[slot]; + + while (e && !inany_equals(&e->addr, addr)) + e = e->next; + + return e; +} + +/** + * fwd_neigh_table_update() - Allocate or update neighbour table entry + * @c: Execution context + * @addr: IP address used to determine insertion slot and store in entry + * @mac: The MAC address associated with the neighbour address + * @permanent: Created entry cannot be altered or freed + */ +void fwd_neigh_table_update(const struct ctx *c, const union inany_addr *addr, + const uint8_t *mac, bool permanent) +{ + struct neigh_table *t = &neigh_table; + struct neigh_table_entry *e; + ssize_t slot; + + /* MAC address might change sometimes */ + e = fwd_neigh_table_find(c, addr); + if (e) { + if (!e->permanent) + memcpy(e->mac, mac, ETH_ALEN); + return; + } + + e = t->free; + if (!e) { + debug("Failed to allocate neighbour table entry"); + return; + } + t->free = e->next; + slot = neigh_table_slot(c, addr); + e->next = t->slots[slot]; + t->slots[slot] = e; + + memcpy(&e->addr, addr, sizeof(*addr)); + memcpy(e->mac, mac, ETH_ALEN); + e->permanent = permanent; +} + +/** + * fwd_neigh_table_free() - Remove an entry from a slot and add it to free list + * @c: Execution context + * @addr: IP address used to find the slot for the entry + */ +void fwd_neigh_table_free(const struct ctx *c, const union inany_addr *addr) +{ + ssize_t slot = neigh_table_slot(c, addr); + struct neigh_table *t = &neigh_table; + struct neigh_table_entry *e, **prev; + + prev = &t->slots[slot]; + e = t->slots[slot]; + while (e && !inany_equals(&e->addr, addr)) { + prev = &e->next; + e = e->next; + } + + if (!e || e->permanent) + return; + + *prev = e->next; + e->next = t->free; + t->free = e; + memset(&e->addr, 0, sizeof(*addr)); + memset(e->mac, 0, ETH_ALEN); +} + +/** + * fwd_neigh_mac_get() - Look up MAC address in the ARP/NDP table + * @c: Execution context + * @addr: Neighbour IP address used as lookup key + * @mac: Buffer for returned MAC address + */ +void fwd_neigh_mac_get(const struct ctx *c, const union inany_addr *addr, + uint8_t *mac) +{ + const struct neigh_table_entry *e = fwd_neigh_table_find(c, addr); + + if (!e) { + union inany_addr ggw; + + if (inany_v4(addr)) + ggw = inany_from_v4(c->ip4.guest_gw); + else + ggw.a6 = c->ip6.guest_gw; + + e = fwd_neigh_table_find(c, &ggw); + } + + if (e) + memcpy(mac, e->mac, ETH_ALEN); + else + memcpy(mac, c->our_tap_mac, ETH_ALEN); +} + +/** + * fwd_neigh_table_init() - Initialize the neighbour table + * @c: Execution context + */ +void fwd_neigh_table_init(const struct ctx *c) +{ + union inany_addr mhl = inany_from_v4(c->ip4.map_host_loopback); + union inany_addr mga = inany_from_v4(c->ip4.map_guest_addr); + struct neigh_table *t = &neigh_table; + struct neigh_table_entry *e; + int i; + + memset(t, 0, sizeof(*t)); + + for (i = 0; i < NEIGH_TABLE_SIZE; i++) { + e = &t->entries[i]; + e->next = t->free; + t->free = e; + } + + /* Blocker entries to stop events from hosts using these addresses */ + if (!inany_is_unspecified4(&mhl)) + fwd_neigh_table_update(c, &mhl, c->our_tap_mac, true); + + if (!inany_is_unspecified4(&mga)) + fwd_neigh_table_update(c, &mga, c->our_tap_mac, true); + + mhl = *(union inany_addr *)&c->ip6.map_host_loopback; + mga = *(union inany_addr *)&c->ip6.map_guest_addr; + + if (!inany_is_unspecified6(&mhl)) + fwd_neigh_table_update(c, &mhl, c->our_tap_mac, true); + + if (!inany_is_unspecified6(&mga)) + fwd_neigh_table_update(c, &mga, c->our_tap_mac, true); +} + /** fwd_probe_ephemeral() - Determine what ports this host considers ephemeral * * Work out what ports the host thinks are emphemeral and record it for later diff --git a/fwd.h b/fwd.h index 65c7c96..352f3b5 100644 --- a/fwd.h +++ b/fwd.h @@ -56,5 +56,12 @@ uint8_t fwd_nat_from_splice(const struct ctx *c, uint8_t proto, const struct flowside *ini, struct flowside *tgt); uint8_t fwd_nat_from_host(const struct ctx *c, uint8_t proto, const struct flowside *ini, struct flowside *tgt); +void fwd_neigh_table_update(const struct ctx *c, const union inany_addr *addr, + const uint8_t *mac, bool permanent); +void fwd_neigh_table_free(const struct ctx *c, + const union inany_addr *addr); +void fwd_neigh_mac_get(const struct ctx *c, const union inany_addr *addr, + uint8_t *mac); +void fwd_neigh_table_init(const struct ctx *c); #endif /* FWD_H */ diff --git a/netlink.c b/netlink.c index 2896e23..ee7466a 100644 --- a/netlink.c +++ b/netlink.c @@ -1123,9 +1123,9 @@ static void nl_neigh_msg_read(const struct ctx *c, struct nlmsghdr *nh) char ip_str[INET6_ADDRSTRLEN]; char mac_str[ETH_ADDRSTRLEN]; const uint8_t *lladdr = NULL; + union inany_addr addr, daddr; const void *dst = NULL; size_t lladdr_len = 0; - union inany_addr addr; size_t dstlen = 0; if (nh->nlmsg_type == NLMSG_DONE) @@ -1172,16 +1172,23 @@ static void nl_neigh_msg_read(const struct ctx *c, struct nlmsghdr *nh) warn("netlink: wrong address length in AF_INET6 notification"); return; } + + /* We only handle guest-side visible addresses */ inany_from_af(&addr, ndm->ndm_family, dst); - inany_ntop(&addr, ip_str, sizeof(ip_str)); + if (!nat_inbound(c, &addr, &daddr)) + return; + + inany_ntop(&daddr, ip_str, sizeof(ip_str)); if (nh->nlmsg_type == RTM_DELNEIGH) { trace("neighbour notifier delete: %s", ip_str); + fwd_neigh_table_free(c, &daddr); return; } if (!(ndm->ndm_state & NUD_VALID)) { trace("neighbour notifier: %s unreachable, state: 0x%02x", ip_str, ndm->ndm_state); + fwd_neigh_table_free(c, &daddr); return; } if (!lladdr) { @@ -1193,6 +1200,7 @@ static void nl_neigh_msg_read(const struct ctx *c, struct nlmsghdr *nh) eth_ntop(lladdr, mac_str, sizeof(mac_str)); trace("neighbour notifier update: %s / %s", ip_str, mac_str); + fwd_neigh_table_update(c, &daddr, lladdr, false); } /** diff --git a/passt.c b/passt.c index a10e469..24e4348 100644 --- a/passt.c +++ b/passt.c @@ -324,6 +324,7 @@ int main(int argc, char **argv) pcap_init(&c); + fwd_neigh_table_init(&c); nl_neigh_notify_init(&c); if (!c.foreground) { -- 2.50.1

3 3

[PATCH v15 1/9] netlink: add subscription on changes in NDP/ARP table
by Jon Maloy 24 Oct '25

24 Oct '25

The solution to bug https://bugs.passt.top/show_bug.cgi?id=120 requires the ability to translate from an IP address to its corresponding MAC address in cases where those are present in the ARP or NDP tables. To keep track of the contents of these tables we add a netlink based neighbour subscription feature. Signed-off-by: Jon Maloy <jmaloy(a)redhat.com> --- v3: - Added an attribute contianing NDA_DST to sent message, so that we let the kernel do the filtering of the IP address and return only one entry. - Added interface index to the call signature. Since the only interface we know is the template interface, this limits the number of hosts that will be seen as 'network segment local' from a PASST viewpoint. v4: - Made loop independent of attribute order. - Ignoring L2 addresses which are not of size ETH_ALEN. v5: - Changed return value of new function, so caller can know if a MAC address really was found. v6: - Removed warning printout which had ended up in the wrong commit. v8: - Changed to neighbour event subscription model - netlink: arp/ndp table subscription v10:- Updated according to David's latest comments on v8 - Added functionaly where we initially read current state of ARP/NDP tables v12:- Updates based on feedback from David and Stefano v13:- Updates based on feedback from David and Stefano v14:- Updates based on feedback from David v15:- Updates based on feedback from Stefano --- epoll_type.h | 2 + netlink.c | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++- netlink.h | 4 + passt.c | 7 ++ 4 files changed, 211 insertions(+), 3 deletions(-) diff --git a/epoll_type.h b/epoll_type.h index 12ac59b..a90ffb6 100644 --- a/epoll_type.h +++ b/epoll_type.h @@ -44,6 +44,8 @@ enum epoll_type { EPOLL_TYPE_REPAIR_LISTEN, /* TCP_REPAIR helper socket */ EPOLL_TYPE_REPAIR, + /* Netlink neighbour subscription socket */ + EPOLL_TYPE_NL_NEIGH, EPOLL_NUM_TYPES, }; diff --git a/netlink.c b/netlink.c index 9fe70f2..2896e23 100644 --- a/netlink.c +++ b/netlink.c @@ -26,6 +26,7 @@ #include <arpa/inet.h> #include <netinet/in.h> #include <netinet/if_ether.h> +#include <net/if_arp.h> #include <linux/netlink.h> #include <linux/rtnetlink.h> @@ -40,6 +41,10 @@ #define RTNH_NEXT_AND_DEC(rtnh, attrlen) \ ((attrlen) -= RTNH_ALIGN((rtnh)->rtnh_len), RTNH_NEXT(rtnh)) +/* Convenience macro borrowed from kernel */ +#define NUD_VALID \ + (NUD_PERMANENT | NUD_NOARP | NUD_REACHABLE | NUD_PROBE | NUD_STALE) + /* Netlink expects a buffer of at least 8kiB or the system page size, * whichever is larger. 32kiB is recommended for more efficient. * Since the largest page size on any remotely common Linux setup is @@ -50,9 +55,10 @@ #define NLBUFSIZ 65536 /* Socket in init, in target namespace, sequence (just needs to be monotonic) */ -int nl_sock = -1; -int nl_sock_ns = -1; -static int nl_seq = 1; +int nl_sock = -1; +int nl_sock_ns = -1; +static int nl_sock_neigh = -1; +static int nl_seq = 1; /** * nl_sock_init_do() - Set up netlink sockets in init or target namespace @@ -1103,3 +1109,192 @@ int nl_link_set_flags(int s, unsigned int ifi, return nl_do(s, &req, RTM_NEWLINK, 0, sizeof(req)); } + +/** + * nl_neigh_msg_read() - Interpret a neighbour state message from netlink + * @c: Execution context + * @nh: Message to be read + */ +static void nl_neigh_msg_read(const struct ctx *c, struct nlmsghdr *nh) +{ + struct ndmsg *ndm = NLMSG_DATA(nh); + struct rtattr *rta = (struct rtattr *)(ndm + 1); + size_t na = NLMSG_PAYLOAD(nh, sizeof(*ndm)); + char ip_str[INET6_ADDRSTRLEN]; + char mac_str[ETH_ADDRSTRLEN]; + const uint8_t *lladdr = NULL; + const void *dst = NULL; + size_t lladdr_len = 0; + union inany_addr addr; + size_t dstlen = 0; + + if (nh->nlmsg_type == NLMSG_DONE) + return; + + if (nh->nlmsg_type == NLMSG_ERROR) { + const struct nlmsgerr *errmsg = (struct nlmsgerr *)ndm; + + warn("netlink error message on neighbour notifier: %s", + strerror_(-errmsg->error)); + return; + } + + if (nh->nlmsg_type != RTM_NEWNEIGH && nh->nlmsg_type != RTM_DELNEIGH) + return; + + for (; RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) { + if (rta->rta_type == NDA_DST) { + dst = RTA_DATA(rta); + dstlen = RTA_PAYLOAD(rta); + } else if (rta->rta_type == NDA_LLADDR) { + lladdr = RTA_DATA(rta); + lladdr_len = RTA_PAYLOAD(rta); + } + } + + if (!dst) + return; + + if (ndm->ndm_family == AF_INET && ndm->ndm_ifindex != c->ifi4) + return; + + if (ndm->ndm_family == AF_INET6 && ndm->ndm_ifindex != c->ifi6) + return; + + if (ndm->ndm_family != AF_INET && ndm->ndm_family != AF_INET6) + return; + + if (ndm->ndm_family == AF_INET && dstlen != sizeof(struct in_addr)) { + warn("netlink: wrong address length in AF_INET notification"); + return; + } + if (ndm->ndm_family == AF_INET6 && dstlen != sizeof(struct in6_addr)) { + warn("netlink: wrong address length in AF_INET6 notification"); + return; + } + inany_from_af(&addr, ndm->ndm_family, dst); + inany_ntop(&addr, ip_str, sizeof(ip_str)); + + if (nh->nlmsg_type == RTM_DELNEIGH) { + trace("neighbour notifier delete: %s", ip_str); + return; + } + if (!(ndm->ndm_state & NUD_VALID)) { + trace("neighbour notifier: %s unreachable, state: 0x%02x", + ip_str, ndm->ndm_state); + return; + } + if (!lladdr) { + warn("RTM_NEWNEIGH %s: missing link layer address", ip_str); + return; + } + if (lladdr_len != ETH_ALEN || ndm->ndm_type != ARPHRD_ETHER) + return; + + eth_ntop(lladdr, mac_str, sizeof(mac_str)); + trace("neighbour notifier update: %s / %s", ip_str, mac_str); +} + +/** + * nl_neigh_sync() - Read current contents of ARP/NDP tables + * @c: Execution context + * @proto: Protocol, AF_INET or AF_INET6 + * @ifi: Interface index + */ +static void nl_neigh_sync(const struct ctx *c, int proto, int ifi) +{ + struct { + struct nlmsghdr nlh; + struct ndmsg ndm; + } req = { + .ndm.ndm_family = proto, + .ndm.ndm_ifindex = ifi, + }; + struct nlmsghdr *nh; + char buf[NLBUFSIZ]; + ssize_t status; + uint32_t seq; + + seq = nl_send(nl_sock_neigh, &req, RTM_GETNEIGH, + NLM_F_DUMP, sizeof(req)); + nl_foreach_oftype(nh, status, nl_sock_neigh, buf, seq, RTM_NEWNEIGH) + nl_neigh_msg_read(c, nh); + if (status < 0) + warn("netlink: RTM_GETNEIGH failed: %s", strerror_(-status)); +} + +/** + * nl_neigh_notify_handler() - Non-blocking drain of pending neighbour updates + * @c: Execution context + */ +void nl_neigh_notify_handler(const struct ctx *c) +{ + char buf[NLBUFSIZ]; + + for (;;) { + ssize_t n = recv(nl_sock_neigh, buf, sizeof(buf), MSG_DONTWAIT); + struct nlmsghdr *nh = (struct nlmsghdr *)buf; + + if (n < 0) { + if (errno == EINTR) + continue; + if (errno != EAGAIN) + warn_perror("netlink notifier read error"); + return; + } + for (; NLMSG_OK(nh, n); nh = NLMSG_NEXT(nh, n)) + nl_neigh_msg_read(c, nh); + } +} + +/** + * nl_neigh_notify_init() - Subscribe to neighbour events + * @c: Execution context + * + * Return: 0 on success, -1 on failure + */ +int nl_neigh_notify_init(const struct ctx *c) +{ + union epoll_ref ref = { + .type = EPOLL_TYPE_NL_NEIGH + }; + struct epoll_event ev = { + .events = EPOLLIN + }; + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + .nl_groups = RTMGRP_NEIGH, + }; + + if (nl_sock_neigh >= 0) { + warn("netlink: neighbour notifier socket already exists"); + return 0; + } + + nl_sock_neigh = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, + NETLINK_ROUTE); + if (nl_sock_neigh < 0) { + warn_perror("Failed to create neighbour notifier socket"); + return -1; + } + + if (bind(nl_sock_neigh, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + warn_perror("Failed to bind neighbour notifier socket"); + close(nl_sock_neigh); + nl_sock_neigh = -1; + return -1; + } + + ev.data.u64 = ref.u64; + if (epoll_ctl(c->epollfd, EPOLL_CTL_ADD, nl_sock_neigh, &ev) == -1) { + warn_perror("epoll_ctl() on neighbour notifier socket failed"); + close(nl_sock_neigh); + nl_sock_neigh = -1; + return -1; + } + + nl_neigh_sync(c, AF_INET, c->ifi4); + nl_neigh_sync(c, AF_INET6, c->ifi6); + + return 0; +} diff --git a/netlink.h b/netlink.h index b51e99c..8f1e9b9 100644 --- a/netlink.h +++ b/netlink.h @@ -17,6 +17,8 @@ int nl_route_dup(int s_src, unsigned int ifi_src, int s_dst, unsigned int ifi_dst, sa_family_t af); int nl_addr_get(int s, unsigned int ifi, sa_family_t af, void *addr, int *prefix_len, void *addr_l); +bool nl_neigh_mac_get(int s, const union inany_addr *addr, int ifi, + unsigned char *mac); int nl_addr_set(int s, unsigned int ifi, sa_family_t af, const void *addr, int prefix_len); int nl_addr_get_ll(int s, unsigned int ifi, struct in6_addr *addr); @@ -28,5 +30,7 @@ int nl_link_set_mac(int s, unsigned int ifi, const void *mac); int nl_link_set_mtu(int s, unsigned int ifi, int mtu); int nl_link_set_flags(int s, unsigned int ifi, unsigned int set, unsigned int change); +int nl_neigh_notify_init(const struct ctx *c); +void nl_neigh_notify_handler(const struct ctx *c); #endif /* NETLINK_H */ diff --git a/passt.c b/passt.c index bdb7b69..a10e469 100644 --- a/passt.c +++ b/passt.c @@ -53,6 +53,7 @@ #include "vu_common.h" #include "migrate.h" #include "repair.h" +#include "netlink.h" #define NUM_EPOLL_EVENTS 8 @@ -79,6 +80,7 @@ char *epoll_type_str[] = { [EPOLL_TYPE_VHOST_KICK] = "vhost-user kick socket", [EPOLL_TYPE_REPAIR_LISTEN] = "TCP_REPAIR helper listening socket", [EPOLL_TYPE_REPAIR] = "TCP_REPAIR helper socket", + [EPOLL_TYPE_NL_NEIGH] = "netlink neighbour notifier socket", }; static_assert(ARRAY_SIZE(epoll_type_str) == EPOLL_NUM_TYPES, "epoll_type_str[] doesn't match enum epoll_type"); @@ -322,6 +324,8 @@ int main(int argc, char **argv) pcap_init(&c); + nl_neigh_notify_init(&c); + if (!c.foreground) { if ((devnull_fd = open("/dev/null", O_RDWR | O_CLOEXEC)) < 0) die_perror("Failed to open /dev/null"); @@ -414,6 +418,9 @@ loop: case EPOLL_TYPE_REPAIR: repair_handler(&c, eventmask); break; + case EPOLL_TYPE_NL_NEIGH: + nl_neigh_notify_handler(&c); + break; default: /* Can't happen */ ASSERT(0); -- 2.50.1

3 2

[PATCH v4 0/7] Refactor epoll handling in preparation for multithreading
by Laurent Vivier 23 Oct '25

23 Oct '25

This series refactors how epoll file descriptors are managed throughout the codebase in preparation for introducing multithreading support. Currently, passt uses a single global epollfd accessed through the context structure. With multithreading, each thread will need its own epollfd managing its subset of flows. The key changes are: 1. Centralize epoll management by extracting helper functions into a new epoll_ctl.c/h module and moving union epoll_ref from passt.h to its more logical location in epoll_ctl.h. 2. Simplify epoll_del() to take the epollfd directly rather than extracting it from the context structure, reducing coupling between epoll operations and the global context. 3. Move epoll registration out of sock_l4_sa() into protocol-specific code, giving callers explicit control over which epoll instance manages each socket. 4. Replace the boolean in_epoll flag in TCP connections with a threadnb (thread number) field in flow_common. This serves dual purposes: tracking registration status (FLOW_THREADNB_INVALID = not registered) and identifying the owning thread. A thread-to-epollfd mapping allows retrieving the actual epoll file descriptor. The threadnb field is 8 bits, limiting values to 0-254 (255 = FLOW_THREADNB_INVALID). 5. Apply this pattern consistently across all protocol handlers (TCP, ICMP, UDP), storing the managing thread number in each flow's common structure. These changes make thread ownership explicit in the flow tracking system, enabling flows to be managed by different threads (and their epoll instances), a prerequisite for per-thread epollfd design in upcoming multithreading work. Changes since v1: - New patch: "epoll_ctl: Extract epoll operations" - centralizes epoll helpers into a dedicated module and relocates union epoll_ref from passt.h to epoll_ctl.h - Changed epollfd type in flow_common from int to 8-bit bitfield to avoid exceeding cacheline size threshold - Added flow_epollfd_valid() helper to check epoll registration status - Added flow_set_epollfd() helper to set the epoll instance for a flow Changes since v2: - Renamed field in flow_common from epollfd to threadnb (thread number) to better reflect that flows are now associated with threads rather than directly with epoll file descriptors - Introduced thread-to-epollfd mapping via threadnb_to_epollfd[] array in flow.c, providing an indirection layer between threads and their epoll instances - Renamed/refactored helper functions: - flow_set_epollfd() -> flow_epollfd_set() - now takes thread number and epollfd parameters - Added flow_epollfd_get() - retrieves the epoll fd for a flow's thread - flow_epollfd_valid() - now checks threadnb instead of epollfd - Updated constants: replaced EPOLLFD_* with FLOW_THREADNB_* equivalents (e.g., EPOLLFD_INVALID -> FLOW_THREADNB_INVALID) - Applied thread-based pattern consistently across TCP, ICMP, and UDP Changes since v3: - Change warn() by err() in epoll_add() - Renamed flow_epollfd_valid() to flow_in_epoll(), flow_epollfd_get() to flow_epollfd(). - Added flow_thread_register() to register an epollfd to a thread number, and called it from main() to register c->epollfd for thread 0 (main loop) - Replaced flow_epollfd_set() by flow_thread_set() to set the thread number of a flow. - Added new patch: "passt: Move main event loop processing into passt_worker()" Laurent Vivier (7): util: Simplify epoll_del() interface to take epollfd directly epoll_ctl: Extract epoll operations util: Move epoll registration out of sock_l4_sa() tcp, flow: Replace per-connection in_epoll flag with threadnb in flow_common icmp: Use thread-based epoll management for ICMP flows udp: Use thread-based epoll management for UDP flows passt: Move main event loop processing into passt_worker() Makefile | 22 +++---- epoll_ctl.c | 45 ++++++++++++++ epoll_ctl.h | 51 ++++++++++++++++ flow.c | 63 +++++++++++++++++--- flow.h | 14 ++++- icmp.c | 23 +++++--- passt.c | 163 ++++++++++++++++++++++++++++----------------------- passt.h | 34 ----------- pasta.c | 7 +-- pif.c | 32 +++++++--- repair.c | 14 ++--- tap.c | 15 ++--- tcp.c | 41 +++++++------ tcp_conn.h | 8 +-- tcp_splice.c | 26 ++++---- udp.c | 2 +- udp_flow.c | 23 ++++++-- util.c | 28 +-------- util.h | 6 +- vhost_user.c | 14 ++--- vu_common.c | 2 +- 21 files changed, 387 insertions(+), 246 deletions(-) create mode 100644 epoll_ctl.c create mode 100644 epoll_ctl.h -- 2.51.0

3 26

[PATCH 3/3] tcp, udp: Bind outbound listening sockets by interface instead of address
by David Gibson 23 Oct '25

23 Oct '25

Currently, outbound forwards (-T, -U) are handled by sockets bound to the loopback address. Typically we create two sockets, one for 127.0.0.1 and one for ::1. This has some disadvantages: * The guest can't connect to these services using its global IP address, it must explicitly use 127.0.0.1 or ::1 (bug 100) * The guest can't even connect via 127.0.0.0/8 addresses other than 127.0.0.1 * We can't use dual-stack sockets, we have to have separate sockets for IPv4 and IPv6. The restriction exist for a reason though. If the guest has any interfaces other than pasta (e.g. a VPN tunnel) external hosts could reach the host via the forwards. Especially combined with -T auto / -U auto this would make it very easy to make a mistake with nasty security implications. We can achieve both goals, however, if we don't bind the outbound listening sockets to a particular address, but _do_ use SO_BINDTODEVICE to restrict them to the "lo" interface. Link: https://bugs.passt.top/show_bug.cgi?id=100 Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au> --- pif.c | 6 ------ tcp.c | 18 ++---------------- udp.c | 27 ++++++++++----------------- 3 files changed, 12 insertions(+), 39 deletions(-) diff --git a/pif.c b/pif.c index 592fafaa..84e3ceae 100644 --- a/pif.c +++ b/pif.c @@ -87,12 +87,6 @@ int pif_sock_l4(const struct ctx *c, enum epoll_type type, uint8_t pif, ASSERT(pif_is_socket(pif)); - if (pif == PIF_SPLICE) { - /* Sanity checks */ - ASSERT(!ifname); - ASSERT(addr && inany_is_loopback(addr)); - } - if (!addr) return sock_l4_sa(c, type, &sa, sizeof(sa.sa6), ifname, false, data); diff --git a/tcp.c b/tcp.c index 15c012d7..982c9190 100644 --- a/tcp.c +++ b/tcp.c @@ -2592,20 +2592,6 @@ int tcp_sock_init(const struct ctx *c, uint8_t pif, return r4 < 0 ? r4 : r6; } -/** - * tcp_ns_sock_init() - Init socket to listen for spliced outbound connections - * @c: Execution context - * @port: Port, host order - */ -static void tcp_ns_sock_init(const struct ctx *c, in_port_t port) -{ - ASSERT(!c->no_tcp); - - if (c->ifi4) - tcp_sock_init_one(c, PIF_SPLICE, &inany_loopback4, NULL, port); - if (c->ifi6) - tcp_sock_init_one(c, PIF_SPLICE, &inany_loopback6, NULL, port); -} /** * tcp_ns_socks_init() - Bind sockets in namespace for outbound connections @@ -2625,7 +2611,7 @@ static int tcp_ns_socks_init(void *arg) if (!bitmap_isset(c->tcp.fwd_out.map, port)) continue; - tcp_ns_sock_init(c, port); + tcp_sock_init(c, PIF_SPLICE, NULL, "lo", port); } return 0; @@ -2805,7 +2791,7 @@ static void tcp_port_rebind(struct ctx *c, bool outbound) if ((c->ifi4 && socks[port][V4] == -1) || (c->ifi6 && socks[port][V6] == -1)) { if (outbound) - tcp_ns_sock_init(c, port); + tcp_sock_init(c, PIF_SPLICE, NULL, "lo", port); else tcp_sock_init(c, PIF_HOST, NULL, NULL, port); } diff --git a/udp.c b/udp.c index 49dd0144..e38114eb 100644 --- a/udp.c +++ b/udp.c @@ -1127,26 +1127,16 @@ int udp_sock_init(const struct ctx *c, uint8_t pif, } if ((!addr || inany_v4(addr)) && c->ifi4) { - const union inany_addr *a = addr ? - addr : &inany_any4; - - if (pif == PIF_SPLICE) - a = &inany_loopback4; - - r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, pif, a, ifname, + r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, pif, + addr ? addr : &inany_any4, ifname, port, uref.u32); socks[V4][port] = r4 < 0 ? -1 : r4; } if ((!addr || !inany_v4(addr)) && c->ifi6) { - const union inany_addr *a = addr ? - addr : &inany_any6; - - if (pif == PIF_SPLICE) - a = &inany_loopback6; - - r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, pif, a, ifname, + r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, pif, + addr ? addr : &inany_any6, ifname, port, uref.u32); socks[V6][port] = r6 < 0 ? -1 : r6; @@ -1214,9 +1204,12 @@ static void udp_port_rebind(struct ctx *c, bool outbound) continue; if ((c->ifi4 && socks[V4][port] == -1) || - (c->ifi6 && socks[V6][port] == -1)) - udp_sock_init(c, outbound ? PIF_SPLICE : PIF_HOST, - NULL, NULL, port); + (c->ifi6 && socks[V6][port] == -1)) { + if (outbound) + udp_sock_init(c, PIF_SPLICE, NULL, "lo", port); + else + udp_sock_init(c, PIF_HOST, NULL, NULL, port); + } } } -- 2.51.0

2 4

[PATCH v14 00/10] Use true MAC address of LAN local remote hosts
by Jon Maloy 22 Oct '25

22 Oct '25

Bug #120 asks us to use the true MAC addresses of LAN local remote hosts, since some programs need this information. These commits introduces this for ARP, NDP, UDP, TCP and ICMP. --- v3: Updated according to feedback from Stefano and David: - Made the ARP/NDP lookup call filter out the requested address by itself, qualified by the index if the template interface - Moved the flow specific MAC address from struct flowside to struct flow_common. v4: - Updated according to feedback from David and Stefan - Added a cache table for ARP/NDP table contents v5: - Updated according to feedback from David and Stefan - Added cache table entries to FIFO/LRU queue - New criteria for when to consult ARP/NDP v6: - Simplified and merged mac cache table commits - Other changes after feedback from David. v7: - Fixes in patch #2 based on feedback from David and Stefano. v8: - Redesigned netlink and cache table part to be based on a subscription model. v8: - Small fix to patch #2 so that we cover the case when a MAC addess for a host has changed. - Added a commit where we send a gratuitous ARP/ unsolicitated NA to the guest when a new host is added to the neighbour cache table. v10: - Some fixes after feedback from David Gibson - Reordered: Moved patch #9 to position #3. - Added synchronization step between ARP/NDP table contents and the neigbour table at initialization. This reduces the number of "false" ARP/NDP replies drastically, but not completly. - (Next step could be to scan over the flow table and update affeced entries when we receive a MAC address update.) v11: - Corrected the gratuitous ARP implementation to use the "ARP Announcement" model instead of the "Gratuitous ARP reply" model. v12: - Updated based on feedback from David and Stefano - Added special handling of default GW and loopback addresses. v13: - Updated based on discussion with David and Stefano - Conceptually moved to only considering guest-side visible addresss. A lot of things became simpler and clearer through this change. Thank you, David. - Introduced a 'permanent' flag in the special entries representing addessed mapping to own host and conditionally the guest gw. This flag indicates those entries cannot be altered by possible remote hosts shadowed by these addresses. Suggested by Stefano. - Reordered patch ##4 and 5, since #5 cannot work correctly for NDP unsolicited NA until #4 is in place. - Added a new commit #2 to get later access to the flag no_map_gw. It was wrong to call fwd_neigh_table_init() from inside conf(), it has to be done in main() after random_init() and tap_backend_init(). v14: - Some fixes after feedback from David Gibson, notably - Moved the call to nat_inbound() from fwd.c to netlink.c - Added RFC quotes to explain the format of ARP announce messages. Jon Maloy (10): netlink: add subscription on changes in NDP/ARP table passt: add no_map_gw flag to struct ctx fwd: Add cache table for ARP/NDP contents arp/ndp: respond with true MAC address of LAN local remote hosts arp/ndp: send ARP announcement / unsolicited NA when neigbour entry added flow: add MAC address of LAN local remote hosts to flow udp: forward external source MAC address through tap interface tcp: forward external source MAC address through tap interface tap: change signature of function tap_push_l2h() icmp: let icmp use mac address from flowside structure arp.c | 58 ++++++++++++- arp.h | 2 + conf.c | 10 +-- epoll_type.h | 2 + flow.c | 2 + flow.h | 2 + fwd.c | 227 +++++++++++++++++++++++++++++++++++++++++++++++++ fwd.h | 7 ++ icmp.c | 8 +- inany.c | 1 + ndp.c | 16 +++- ndp.h | 1 + netlink.c | 215 +++++++++++++++++++++++++++++++++++++++++++++- netlink.h | 4 + passt.c | 17 ++-- passt.h | 4 +- pasta.c | 2 +- tap.c | 24 +++--- tap.h | 7 +- tcp.c | 20 ++++- tcp.h | 2 +- tcp_buf.c | 37 ++++---- tcp_internal.h | 4 +- tcp_vu.c | 5 +- udp.c | 57 ++++++++----- udp.h | 2 +- util.h | 2 + 27 files changed, 650 insertions(+), 88 deletions(-) -- 2.50.1

3 24

[PATCH 2/3] udp: Unify some more inbound/outbound parts of udp_sock_init()
by David Gibson 22 Oct '25

22 Oct '25

udp_sock_init() takes an 'ns' parameter determining if it creates a socket in the guest namespace or host namespace. Alter it to take a pif parameter instead, like tcp_sock_init(), and use that change to slightly reduce code duplication between the HOST and SPLICE cases. Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au> --- conf.c | 2 +- udp.c | 60 +++++++++++++++++++++++++++++----------------------------- udp.h | 5 +++-- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/conf.c b/conf.c index 26f1bcc0..08cb50aa 100644 --- a/conf.c +++ b/conf.c @@ -171,7 +171,7 @@ static void conf_ports_range_except(const struct ctx *c, char optname, if (optname == 't') ret = tcp_sock_init(c, PIF_HOST, addr, ifname, i); else if (optname == 'u') - ret = udp_sock_init(c, 0, addr, ifname, i); + ret = udp_sock_init(c, PIF_HOST, addr, ifname, i); else /* No way to check in advance for -T and -U */ ret = 0; diff --git a/udp.c b/udp.c index 86585b7e..49dd0144 100644 --- a/udp.c +++ b/udp.c @@ -1093,64 +1093,63 @@ int udp_tap_handler(const struct ctx *c, uint8_t pif, /** * udp_sock_init() - Initialise listening sockets for a given port * @c: Execution context - * @ns: In pasta mode, if set, bind with loopback address in namespace + * @pif: Interface to open the socket for (PIF_HOST or PIF_SPLICE) * @addr: Pointer to address for binding, NULL if not configured * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order * * Return: 0 on (partial) success, negative error code on (complete) failure */ -int udp_sock_init(const struct ctx *c, int ns, const union inany_addr *addr, - const char *ifname, in_port_t port) +int udp_sock_init(const struct ctx *c, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port) { + int (*socks)[NUM_PORTS] = pif == PIF_HOST ? + udp_splice_init : udp_splice_ns; union udp_listen_epoll_ref uref = { - .pif = ns ? PIF_SPLICE : PIF_HOST, + .pif = pif, .port = port, }; int r4 = FD_REF_MAX + 1, r6 = FD_REF_MAX + 1; ASSERT(!c->no_udp); - if (!addr && c->ifi4 && c->ifi6 && !ns) { + if (!addr && c->ifi4 && c->ifi6 && (pif == PIF_HOST)) { int s; /* Attempt to get a dual stack socket */ s = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_HOST, NULL, ifname, port, uref.u32); - udp_splice_init[V4][port] = s < 0 ? -1 : s; - udp_splice_init[V6][port] = s < 0 ? -1 : s; + socks[V4][port] = s < 0 ? -1 : s; + socks[V6][port] = s < 0 ? -1 : s; if (IN_INTERVAL(0, FD_REF_MAX, s)) return 0; } if ((!addr || inany_v4(addr)) && c->ifi4) { - if (!ns) { - r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_HOST, - addr ? addr : &inany_any4, ifname, - port, uref.u32); + const union inany_addr *a = addr ? + addr : &inany_any4; - udp_splice_init[V4][port] = r4 < 0 ? -1 : r4; - } else { - r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_SPLICE, - &inany_loopback4, ifname, - port, uref.u32); - udp_splice_ns[V4][port] = r4 < 0 ? -1 : r4; - } + if (pif == PIF_SPLICE) + a = &inany_loopback4; + + r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, pif, a, ifname, + port, uref.u32); + + socks[V4][port] = r4 < 0 ? -1 : r4; } if ((!addr || !inany_v4(addr)) && c->ifi6) { - if (!ns) { - r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_HOST, - addr ? addr : &inany_any6, ifname, - port, uref.u32); + const union inany_addr *a = addr ? + addr : &inany_any6; - udp_splice_init[V6][port] = r6 < 0 ? -1 : r6; - } else { - r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_SPLICE, - &inany_loopback6, ifname, - port, uref.u32); - udp_splice_ns[V6][port] = r6 < 0 ? -1 : r6; - } + if (pif == PIF_SPLICE) + a = &inany_loopback6; + + r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, pif, a, ifname, + port, uref.u32); + + socks[V6][port] = r6 < 0 ? -1 : r6; } if (IN_INTERVAL(0, FD_REF_MAX, r4) || IN_INTERVAL(0, FD_REF_MAX, r6)) @@ -1216,7 +1215,8 @@ static void udp_port_rebind(struct ctx *c, bool outbound) if ((c->ifi4 && socks[V4][port] == -1) || (c->ifi6 && socks[V6][port] == -1)) - udp_sock_init(c, outbound, NULL, NULL, port); + udp_sock_init(c, outbound ? PIF_SPLICE : PIF_HOST, + NULL, NULL, port); } } diff --git a/udp.h b/udp.h index 8f8531ad..f78dc528 100644 --- a/udp.h +++ b/udp.h @@ -17,8 +17,9 @@ int udp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, uint8_t ttl, const struct pool *p, int idx, const struct timespec *now); -int udp_sock_init(const struct ctx *c, int ns, const union inany_addr *addr, - const char *ifname, in_port_t port); +int udp_sock_init(const struct ctx *c, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port); int udp_init(struct ctx *c); void udp_timer(struct ctx *c, const struct timespec *now); void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s); -- 2.51.0

2 2

[PATCH 1/3] tcp: Merge tcp_ns_sock_init[46]() into tcp_sock_init_one()
by David Gibson 20 Oct '25

20 Oct '25

Surprisingly little logic is shared between the path for creating a listen()ing socket in the guest namespace versus in the host namespace. Improve this, by extending tcp_sock_init_one() to take a pif parameter indicating where it should open the socket. This allows tcp_ns_sock_init[46]() to be removed entirely. We generalise tcp_sock_init() in the same way, although we don't use it yet, due to some subtle differences in how we bind for -t versus -T. Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au> --- conf.c | 2 +- tcp.c | 96 ++++++++++++++++++---------------------------------------- tcp.h | 5 +-- 3 files changed, 33 insertions(+), 70 deletions(-) diff --git a/conf.c b/conf.c index 66b9e634..26f1bcc0 100644 --- a/conf.c +++ b/conf.c @@ -169,7 +169,7 @@ static void conf_ports_range_except(const struct ctx *c, char optname, fwd->delta[i] = to - first; if (optname == 't') - ret = tcp_sock_init(c, addr, ifname, i); + ret = tcp_sock_init(c, PIF_HOST, addr, ifname, i); else if (optname == 'u') ret = udp_sock_init(c, 0, addr, ifname, i); else diff --git a/tcp.c b/tcp.c index 0f9e9b3f..15c012d7 100644 --- a/tcp.c +++ b/tcp.c @@ -2515,29 +2515,38 @@ void tcp_sock_handler(const struct ctx *c, union epoll_ref ref, /** * tcp_sock_init_one() - Initialise listening socket for address and port * @c: Execution context + * @pif: Interface to open the socket for (PIF_HOST or PIF_SPLICE) * @addr: Pointer to address for binding, NULL for dual stack any * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order * * Return: fd for the new listening socket, negative error code on failure + * + * If pif == PIF_SPLICE, must have already entered the namespace. */ -static int tcp_sock_init_one(const struct ctx *c, const union inany_addr *addr, - const char *ifname, in_port_t port) +static int tcp_sock_init_one(const struct ctx *c, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port) { + const struct fwd_ports *fwd = pif == PIF_HOST ? + &c->tcp.fwd_in : &c->tcp.fwd_out; union tcp_listen_epoll_ref tref = { .port = port, - .pif = PIF_HOST, + .pif = pif, }; int s; - s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, PIF_HOST, addr, - ifname, port, tref.u32); + s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, pif, addr, ifname, + port, tref.u32); + + if (fwd->mode == FWD_AUTO) { + int (*socks)[IP_VERSIONS] = pif == PIF_SPLICE ? + tcp_sock_ns : tcp_sock_init_ext; - if (c->tcp.fwd_in.mode == FWD_AUTO) { if (!addr || inany_v4(addr)) - tcp_sock_init_ext[port][V4] = s < 0 ? -1 : s; + socks[port][V4] = s < 0 ? -1 : s; if (!addr || !inany_v4(addr)) - tcp_sock_init_ext[port][V6] = s < 0 ? -1 : s; + socks[port][V6] = s < 0 ? -1 : s; } if (s < 0) @@ -2549,14 +2558,16 @@ static int tcp_sock_init_one(const struct ctx *c, const union inany_addr *addr, /** * tcp_sock_init() - Create listening sockets for a given host ("inbound") port * @c: Execution context + * @pif: Interface to open the socket for (PIF_HOST or PIF_SPLICE) * @addr: Pointer to address for binding, NULL if not configured * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order * * Return: 0 on (partial) success, negative error code on (complete) failure */ -int tcp_sock_init(const struct ctx *c, const union inany_addr *addr, - const char *ifname, in_port_t port) +int tcp_sock_init(const struct ctx *c, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port) { int r4 = FD_REF_MAX + 1, r6 = FD_REF_MAX + 1; @@ -2564,72 +2575,23 @@ int tcp_sock_init(const struct ctx *c, const union inany_addr *addr, if (!addr && c->ifi4 && c->ifi6) /* Attempt to get a dual stack socket */ - if (tcp_sock_init_one(c, NULL, ifname, port) >= 0) + if (tcp_sock_init_one(c, pif, NULL, ifname, port) >= 0) return 0; /* Otherwise create a socket per IP version */ if ((!addr || inany_v4(addr)) && c->ifi4) - r4 = tcp_sock_init_one(c, addr ? addr : &inany_any4, - ifname, port); + r4 = tcp_sock_init_one(c, pif, + addr ? addr : &inany_any4, ifname, port); if ((!addr || !inany_v4(addr)) && c->ifi6) - r6 = tcp_sock_init_one(c, addr ? addr : &inany_any6, - ifname, port); + r6 = tcp_sock_init_one(c, pif, + addr ? addr : &inany_any6, ifname, port); if (IN_INTERVAL(0, FD_REF_MAX, r4) || IN_INTERVAL(0, FD_REF_MAX, r6)) return 0; return r4 < 0 ? r4 : r6; } - -/** - * tcp_ns_sock_init4() - Init socket to listen for outbound IPv4 connections - * @c: Execution context - * @port: Port, host order - */ -static void tcp_ns_sock_init4(const struct ctx *c, in_port_t port) -{ - union tcp_listen_epoll_ref tref = { - .port = port, - .pif = PIF_SPLICE, - }; - int s; - - ASSERT(c->mode == MODE_PASTA); - - s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, PIF_SPLICE, &inany_loopback4, - NULL, port, tref.u32); - if (s < 0) - s = -1; - - if (c->tcp.fwd_out.mode == FWD_AUTO) - tcp_sock_ns[port][V4] = s; -} - -/** - * tcp_ns_sock_init6() - Init socket to listen for outbound IPv6 connections - * @c: Execution context - * @port: Port, host order - */ -static void tcp_ns_sock_init6(const struct ctx *c, in_port_t port) -{ - union tcp_listen_epoll_ref tref = { - .port = port, - .pif = PIF_SPLICE, - }; - int s; - - ASSERT(c->mode == MODE_PASTA); - - s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, PIF_SPLICE, &inany_loopback6, - NULL, port, tref.u32); - if (s < 0) - s = -1; - - if (c->tcp.fwd_out.mode == FWD_AUTO) - tcp_sock_ns[port][V6] = s; -} - /** * tcp_ns_sock_init() - Init socket to listen for spliced outbound connections * @c: Execution context @@ -2640,9 +2602,9 @@ static void tcp_ns_sock_init(const struct ctx *c, in_port_t port) ASSERT(!c->no_tcp); if (c->ifi4) - tcp_ns_sock_init4(c, port); + tcp_sock_init_one(c, PIF_SPLICE, &inany_loopback4, NULL, port); if (c->ifi6) - tcp_ns_sock_init6(c, port); + tcp_sock_init_one(c, PIF_SPLICE, &inany_loopback6, NULL, port); } /** @@ -2845,7 +2807,7 @@ static void tcp_port_rebind(struct ctx *c, bool outbound) if (outbound) tcp_ns_sock_init(c, port); else - tcp_sock_init(c, NULL, NULL, port); + tcp_sock_init(c, PIF_HOST, NULL, NULL, port); } } } diff --git a/tcp.h b/tcp.h index 234a8033..fb22bac0 100644 --- a/tcp.h +++ b/tcp.h @@ -18,8 +18,9 @@ void tcp_sock_handler(const struct ctx *c, union epoll_ref ref, int tcp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, uint32_t flow_lbl, const struct pool *p, int idx, const struct timespec *now); -int tcp_sock_init(const struct ctx *c, const union inany_addr *addr, - const char *ifname, in_port_t port); +int tcp_sock_init(const struct ctx *c, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port); int tcp_init(struct ctx *c); void tcp_timer(struct ctx *c, const struct timespec *now); void tcp_defer_handler(struct ctx *c); -- 2.51.0

2 4

[PATCH v4 0/4] Retry SYNs for inbound connections
by Yumei Huang 19 Oct '25

19 Oct '25

When a client connects, SYN would be sent to guest only once. If the guest is not connected or ready at that time, the connection will be reset in 10s. These patches introduce the SYN retry mechanism using the similar backoff timeout as linux kernel. Also update the data retransmission timeout using the backoff timeout. Yumei Huang (4): tcp: Rename "retrans" to "retries" util: Introduce read_file() and read_file_integer() function tcp: Resend SYN for inbound connections tcp: Update data retransmission timeout tcp.c | 74 ++++++++++++++++++++++++++++++++++------------- tcp.h | 2 ++ tcp_conn.h | 12 ++++---- util.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ util.h | 3 ++ 5 files changed, 149 insertions(+), 26 deletions(-) -- 2.47.0

3 16

[PATCH v5 3/4] tcp: Resend SYN for inbound connections
by Yumei Huang 17 Oct '25

17 Oct '25

If a client connects while guest is not connected or ready yet, resend SYN instead of just resetting connection after 10 seconds. Use the same backoff calculation for the timeout as linux kernel. Link: https://bugs.passt.top/show_bug.cgi?id=153 Signed-off-by: Yumei Huang <yuhuang(a)redhat.com> --- tcp.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++-------- tcp.h | 5 +++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/tcp.c b/tcp.c index 2ec4b0c..71e2da0 100644 --- a/tcp.c +++ b/tcp.c @@ -179,9 +179,11 @@ * * Timeouts are implemented by means of timerfd timers, set based on flags: * - * - SYN_TIMEOUT: if no ACK is received from tap/guest during handshake (flag - * ACK_FROM_TAP_DUE without ESTABLISHED event) within this time, reset the - * connection + * - SYN_TIMEOUT_INIT: if no ACK is received from tap/guest during handshake + * (flag ACK_FROM_TAP_DUE without ESTABLISHED event) within this time, resend + * SYN. It's the starting timeout for the first SYN retry. If this persists + * for more than TCP_MAX_RETRIES or (tcp_syn_retries + + * tcp_syn_linear_timeouts) times in a row, reset the connection * * - ACK_TIMEOUT: if no ACK segment was received from tap/guest, after sending * data (flag ACK_FROM_TAP_DUE with ESTABLISHED event), re-send data from the @@ -340,7 +342,7 @@ enum { #define WINDOW_DEFAULT 14600 /* RFC 6928 */ #define ACK_INTERVAL 10 /* ms */ -#define SYN_TIMEOUT 10 /* s */ +#define SYN_TIMEOUT_INIT 1 /* s */ #define ACK_TIMEOUT 2 #define FIN_TIMEOUT 60 #define ACT_TIMEOUT 7200 @@ -365,6 +367,9 @@ uint8_t tcp_migrate_rcv_queue [TCP_MIGRATE_RCV_QUEUE_MAX]; #define TCP_MIGRATE_RESTORE_CHUNK_MIN 1024 /* Try smaller when above this */ +#define TCP_SYN_RETRIES "/proc/sys/net/ipv4/tcp_syn_retries" +#define TCP_SYN_LINEAR_TIMEOUTS "/proc/sys/net/ipv4/tcp_syn_linear_timeouts" \ + /* "Extended" data (not stored in the flow table) for TCP flow migration */ static struct tcp_tap_transfer_ext migrate_ext[FLOW_MAX]; @@ -581,8 +586,13 @@ static void tcp_timer_ctl(const struct ctx *c, struct tcp_tap_conn *conn) if (conn->flags & ACK_TO_TAP_DUE) { it.it_value.tv_nsec = (long)ACK_INTERVAL * 1000 * 1000; } else if (conn->flags & ACK_FROM_TAP_DUE) { - if (!(conn->events & ESTABLISHED)) - it.it_value.tv_sec = SYN_TIMEOUT; + if (!(conn->events & ESTABLISHED)) { + if (conn->retries < c->tcp.syn_linear_timeouts) + it.it_value.tv_sec = SYN_TIMEOUT_INIT; + else + it.it_value.tv_sec = SYN_TIMEOUT_INIT << + (conn->retries - c->tcp.syn_linear_timeouts); + } else it.it_value.tv_sec = ACK_TIMEOUT; } else if (CONN_HAS(conn, SOCK_FIN_SENT | TAP_FIN_ACKED)) { @@ -2409,8 +2419,17 @@ void tcp_timer_handler(const struct ctx *c, union epoll_ref ref) tcp_timer_ctl(c, conn); } else if (conn->flags & ACK_FROM_TAP_DUE) { if (!(conn->events & ESTABLISHED)) { - flow_dbg(conn, "handshake timeout"); - tcp_rst(c, conn); + if (conn->retries >= TCP_MAX_RETRIES || + conn->retries >= (c->tcp.tcp_syn_retries + + c->tcp.syn_linear_timeouts)) { + flow_dbg(conn, "handshake timeout"); + tcp_rst(c, conn); + } else { + flow_trace(conn, "SYN timeout, retry"); + tcp_send_flag(c, conn, SYN); + conn->retries++; + tcp_timer_ctl(c, conn); + } } else if (CONN_HAS(conn, SOCK_FIN_SENT | TAP_FIN_ACKED)) { flow_dbg(conn, "FIN timeout"); tcp_rst(c, conn); @@ -2766,6 +2785,24 @@ static socklen_t tcp_probe_tcp_info(void) return sl; } +/** + * tcp_syn_params_init() - Get initial syn params for inbound connection + * @c: Execution context +*/ +void tcp_syn_params_init(struct ctx *c) +{ + intmax_t tcp_syn_retries, syn_linear_timeouts; + + tcp_syn_retries = read_file_integer(TCP_SYN_RETRIES, 8); + syn_linear_timeouts = read_file_integer(TCP_SYN_LINEAR_TIMEOUTS, 1); + + c->tcp.tcp_syn_retries = MIN(tcp_syn_retries, UINT8_MAX); + c->tcp.syn_linear_timeouts = MIN(syn_linear_timeouts, UINT8_MAX); + + debug("TCP SYN parameters: retries=%"PRIu8", linear_timeouts=%"PRIu8, + c->tcp.tcp_syn_retries, c->tcp.syn_linear_timeouts); +} + /** * tcp_init() - Get initial sequence, hash secret, initialise per-socket data * @c: Execution context @@ -2776,6 +2813,8 @@ int tcp_init(struct ctx *c) { ASSERT(!c->no_tcp); + tcp_syn_params_init(c); + tcp_sock_iov_init(c); memset(init_sock_pool4, 0xff, sizeof(init_sock_pool4)); diff --git a/tcp.h b/tcp.h index 234a803..bb58324 100644 --- a/tcp.h +++ b/tcp.h @@ -59,12 +59,17 @@ union tcp_listen_epoll_ref { * @fwd_out: Port forwarding configuration for outbound packets * @timer_run: Timestamp of most recent timer run * @pipe_size: Size of pipes for spliced connections + * @tcp_syn_retries: Number of times initial SYNs during handshake + * @syn_linear_timeouts: Number of SYN retries using linear backoff timeout + * before switching to exponential backoff timeout */ struct tcp_ctx { struct fwd_ports fwd_in; struct fwd_ports fwd_out; struct timespec timer_run; size_t pipe_size; + uint8_t tcp_syn_retries; + uint8_t syn_linear_timeouts; }; #endif /* TCP_H */ -- 2.47.0

2 4

[PATCH v5 2/4] util: Introduce read_file() and read_file_integer() function
by Yumei Huang 17 Oct '25

17 Oct '25

Signed-off-by: Yumei Huang <yuhuang(a)redhat.com> --- util.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ util.h | 10 +++++++ 2 files changed, 94 insertions(+) diff --git a/util.c b/util.c index c492f90..ff21bf6 100644 --- a/util.c +++ b/util.c @@ -579,6 +579,90 @@ int write_file(const char *path, const char *buf) return len == 0 ? 0 : -1; } +/** + * read_file() - Read contents of file into a buffer + * @path: File to read + * @buf: Buffer to store file contents + * @buf_size: Size of buffer + * + * Return: number of bytes read on success, -1 on any error, -2 on truncation +*/ +ssize_t read_file(const char *path, char *buf, size_t buf_size) +{ + int fd = open(path, O_RDONLY | O_CLOEXEC); + size_t total_read = 0; + ssize_t rc; + + if (fd < 0) { + warn_perror("Could not open %s", path); + return -1; + } + + while (total_read < buf_size) { + rc = read(fd, buf + total_read, buf_size - total_read); + + if (rc < 0) { + warn_perror("Couldn't read from %s", path); + close(fd); + return -1; + } + + if (rc == 0) + break; + + total_read += rc; + } + + close(fd); + + if (total_read == buf_size) { + warn_perror("File %s truncated, buffer too small", path); + return -2; + } + + buf[total_read] = '\0'; + + return total_read; +} + +/** + * read_file_integer() - Read an integer value from a file + * @path: File to read + * @fallback: Default value if file can't be read + * + * Return: Integer value, fallback on failure +*/ +intmax_t read_file_integer(const char *path, intmax_t fallback) +{ + char buf[INTMAX_STRLEN]; + ssize_t bytes_read; + intmax_t value; + char *end; + + bytes_read = read_file(path, buf, sizeof(buf)); + + if (bytes_read < 0) + return fallback; + + if (bytes_read == 0) { + debug("Empty file %s", path); + return fallback; + } + + errno = 0; + value = strtoimax(buf, &end, 10); + if (*end && *end != '\n') { + debug("Invalid format in %s", path); + return fallback; + } + if (errno) { + debug("Invalid value in %s: %s", path, buf); + return fallback; + } + + return value; +} + #ifdef __ia64__ /* Needed by do_clone() below: glibc doesn't export the prototype of __clone2(), * use the description from clone(2). diff --git a/util.h b/util.h index 22eaac5..acfbd08 100644 --- a/util.h +++ b/util.h @@ -222,6 +222,8 @@ void pidfile_write(int fd, pid_t pid); int __daemon(int pidfile_fd, int devnull_fd); int fls(unsigned long x); int write_file(const char *path, const char *buf); +ssize_t read_file(const char *path, char *buf, size_t buf_size); +intmax_t read_file_integer(const char *path, intmax_t fallback); int write_all_buf(int fd, const void *buf, size_t len); int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, size_t skip); int read_all_buf(int fd, void *buf, size_t len); @@ -250,6 +252,14 @@ static inline const char *af_name(sa_family_t af) #define UINT16_STRLEN (sizeof("65535")) +/* Each byte holds roughly 2.4 decimal digits: + * 1 bit = log10(2) ~= 0.30103 decimal digits + * 1 byte = 8 bits = 8 x 0.30103 ~= 2.408 decimal digits + * Using sizeof(intmax_t) * 3 provides a safe upper bound, + * plus 2 extra bytes for the sign and null terminator. + */ +#define INTMAX_STRLEN (sizeof(intmax_t) * 3 + 2) + /* inet address (- '\0') + port (u16) (- '\0') + ':' + '\0' */ #define SOCKADDR_INET_STRLEN \ (INET_ADDRSTRLEN-1 + UINT16_STRLEN-1 + sizeof(":")) -- 2.47.0

2 1