- dev - passt

[PATCH 00/10] Fixes and cleanups for capability handling
by David Gibson 14 Oct '22

14 Oct '22

Our current handling of capabilities isn't quite right. In particular, drop_caps() attempts to remove capabilities from the bounding set, which usually won't work, and even if it does won't have the effect we want. This series corrects that, as well as making some other fixes and cleanups in adjacent code. David Gibson (10): test: Move slower tests to end of test run pasta: More general way of starting spawned shell as a login shell pasta_start_ns() always ends in parent context Remove unhelpful drop_caps() call in pasta_start_ns() Clarify various self-isolation steps Replace FWRITE with a function isolation: Replace drop_caps() with a version that actually does something isolation: Prevent any child processes gaining capabilities isolation: Only configure UID/GID mappings in userns when spawning shell Rename pasta_setup_ns() to pasta_spawn_cmd() conf.c | 3 +- isolation.c | 199 ++++++++++++++++++++++++++++++++++++++++++++++------ isolation.h | 6 +- passt.c | 8 +-- pasta.c | 72 +++++++++++-------- pasta.h | 3 +- test/run | 20 +++--- util.c | 33 +++++++++ util.h | 13 +--- 9 files changed, 275 insertions(+), 82 deletions(-) -- 2.37.3

2 32

[PATCH v4] conf, tcp, udp: Allow specification of interface to bind to
by Stefano Brivio 13 Oct '22

13 Oct '22

Since kernel version 5.7, commit c427bfec18f2 ("net: core: enable SO_BINDTODEVICE for non-root users"), we can bind sockets to interfaces, if they haven't been bound yet (as in bind()). Introduce an optional interface specification for forwarded ports, prefixed by %, that can be passed together with an address. Reported use case: running local services that use ports we want to have externally forwarded: https://github.com/containers/podman/issues/14425 Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- v4: - use interface also in exclude specifications (David Gibson) v3: - escape % characters in usage() formatting v2: - fix check on interface name length (spec - ifname, not ifname - buf) conf.c | 31 +++++++++++++++++++++---------- icmp.c | 4 ++-- passt.1 | 12 ++++++++++-- tcp.c | 27 +++++++++++++++------------ tcp.h | 2 +- udp.c | 35 ++++++++++++++++++----------------- udp.h | 2 +- util.c | 19 ++++++++++++++++++- util.h | 3 ++- 9 files changed, 88 insertions(+), 47 deletions(-) diff --git a/conf.c b/conf.c index 779371f..56f33f0 100644 --- a/conf.c +++ b/conf.c @@ -180,8 +180,8 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, struct port_fwd *fwd) { char addr_buf[sizeof(struct in6_addr)] = { 0 }, *addr = addr_buf; + char buf[BUFSIZ], *spec, *ifname = NULL, *p; uint8_t exclude[PORT_BITMAP_SIZE] = { 0 }; - char buf[BUFSIZ], *spec, *p; sa_family_t af = AF_UNSPEC; bool exclude_only = true; @@ -209,9 +209,9 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, for (i = 0; i < PORT_EPHEMERAL_MIN; i++) { if (optname == 't') - tcp_sock_init(c, 0, AF_UNSPEC, NULL, i); + tcp_sock_init(c, 0, AF_UNSPEC, NULL, NULL, i); else if (optname == 'u') - udp_sock_init(c, 0, AF_UNSPEC, NULL, i); + udp_sock_init(c, 0, AF_UNSPEC, NULL, NULL, i); } return 0; @@ -231,6 +231,14 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, if (optname != 't' && optname != 'u') goto bad; + if ((ifname = strchr(buf, '%'))) { + if (spec - ifname >= IFNAMSIZ - 1) + goto bad; + + *ifname = 0; + ifname++; + } + if (inet_pton(AF_INET, buf, addr)) af = AF_INET; else if (inet_pton(AF_INET6, buf, addr)) @@ -278,9 +286,9 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, bitmap_set(fwd->map, i); if (optname == 't') - tcp_sock_init(c, 0, af, addr, i); + tcp_sock_init(c, 0, af, addr, ifname, i); else if (optname == 'u') - udp_sock_init(c, 0, af, addr, i); + udp_sock_init(c, 0, af, addr, ifname, i); } return 0; @@ -324,9 +332,9 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, fwd->delta[i] = mapped_range.first - orig_range.first; if (optname == 't') - tcp_sock_init(c, 0, af, addr, i); + tcp_sock_init(c, 0, af, addr, ifname, i); else if (optname == 'u') - udp_sock_init(c, 0, af, addr, i); + udp_sock_init(c, 0, af, addr, ifname, i); } } while ((p = next_chunk(p, ','))); @@ -720,8 +728,9 @@ static void usage(const char *name) info( " 'all': forward all unbound, non-ephemeral ports"); info( " a comma-separated list, optionally ranged with '-'"); info( " and optional target ports after ':', with optional"); - info( " address specification suffixed by '/'. Ranges can be"); - info( " reduced by excluding ports or ranges prefixed by '~'"); + info( " address specification suffixed by '/' and optional"); + info( " interface prefixed by '%%'. Ranges can be reduced by"); + info( " excluding ports or ranges prefixed by '~'"); info( " Examples:"); info( " -t 22 Forward local port 22 to 22 on guest"); info( " -t 22:23 Forward local port 22 to 23 on guest"); @@ -740,6 +749,7 @@ static void usage(const char *name) exit(EXIT_FAILURE); pasta_opts: + info( " -t, --tcp-ports SPEC TCP port forwarding to namespace"); info( " can be specified multiple times"); info( " SPEC can be:"); @@ -747,7 +757,8 @@ pasta_opts: info( " 'auto': forward all ports currently bound in namespace"); info( " a comma-separated list, optionally ranged with '-'"); info( " and optional target ports after ':', with optional"); - info( " address specification suffixed by '/'. Examples:"); + info( " address specification suffixed by '/' and optional"); + info( " interface prefixed by '%%'. Examples:"); info( " -t 22 Forward local port 22 to port 22 in netns"); info( " -t 22:23 Forward local port 22 to port 23"); info( " -t 22,25 Forward ports 22, 25 to ports 22, 25"); diff --git a/icmp.c b/icmp.c index f02f89f..6bd87fd 100644 --- a/icmp.c +++ b/icmp.c @@ -169,7 +169,7 @@ int icmp_tap_handler(const struct ctx *c, int af, const void *addr, iref.icmp.id = id = ntohs(ih->un.echo.id); if ((s = icmp_id_map[V4][id].sock) <= 0) { - s = sock_l4(c, AF_INET, IPPROTO_ICMP, NULL, id, + s = sock_l4(c, AF_INET, IPPROTO_ICMP, NULL, NULL, id, iref.u32); if (s < 0) goto fail_sock; @@ -207,7 +207,7 @@ int icmp_tap_handler(const struct ctx *c, int af, const void *addr, iref.icmp.id = id = ntohs(ih->icmp6_identifier); if ((s = icmp_id_map[V6][id].sock) <= 0) { - s = sock_l4(c, AF_INET6, IPPROTO_ICMPV6, NULL, id, + s = sock_l4(c, AF_INET6, IPPROTO_ICMPV6, NULL, NULL, id, iref.u32); if (s < 0) goto fail_sock; diff --git a/passt.1 b/passt.1 index 555a50c..7d113f2 100644 --- a/passt.1 +++ b/passt.1 @@ -325,7 +325,8 @@ For low (< 1024) ports, see \fBNOTES\fR. .BR ports A comma-separated list of ports, optionally ranged with \fI-\fR, and, optionally, with target ports after \fI:\fR, if they differ. Specific addresses -can be bound as well, separated by \fI/\fR. Within given ranges, selected ports +can be bound as well, separated by \fI/\fR, and also, since Linux 5.7, limited +to specific interfaces, prefixed by \fI%\fR. Within given ranges, selected ports and ranges can be excluded by an additional specification prefixed by \fI~\fR. Specifying excluded ranges only implies that all other ports are forwarded. Examples: @@ -349,6 +350,9 @@ Forward local ports 22 to 80 to corresponding ports on the guest plus 10 -t 192.0.2.1/22 Forward local port 22, bound to 192.0.2.1, to port 22 on the guest .TP +-t 192.0.2.1%eth0/22 +Forward local port 22, bound to 192.0.2.1 and interface eth0, to port 22 +.TP -t 2000-5000,~3000-3010 Forward local ports 2000 to 5000, but not 3000 to 3010 .TP @@ -399,7 +403,8 @@ periodically derived (every second) from listening sockets reported by .BR ports A comma-separated list of ports, optionally ranged with \fI-\fR, and, optionally, with target ports after \fI:\fR, if they differ. Specific addresses -can be bound as well, separated by \fI/\fR. Within given ranges, selected ports +can be bound as well, separated by \fI/\fR, and also, since Linux 5.7, limited +to specific interfaces, prefixed by \fI%\fR. Within given ranges, selected ports and ranges can be excluded by an additional specification prefixed by \fI~\fR. Specifying excluded ranges only implies that all other ports are forwarded. Examples: @@ -424,6 +429,9 @@ namespace -t 192.0.2.1/22 Forward local port 22, bound to 192.0.2.1, to port 22 in the target namespace .TP +-t 192.0.2.1%eth0/22 +Forward local port 22, bound to 192.0.2.1 and interface eth0, to port 22 +.TP -t 2000-5000,~3000-3010 Forward local ports 2000 to 5000, but not 3000 to 3010 .TP diff --git a/tcp.c b/tcp.c index 830dc88..7e82589 100644 --- a/tcp.c +++ b/tcp.c @@ -3078,10 +3078,11 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events, * @ns: In pasta mode, if set, bind with loopback address in namespace * @af: Address family to select a specific IP version, or AF_UNSPEC * @addr: Pointer to address for binding, NULL if not configured + * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order */ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port) + const void *addr, const char *ifname, in_port_t port) { union tcp_epoll_ref tref = { .tcp.listen = 1 }; const void *bind_addr; @@ -3103,8 +3104,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, tref.tcp.splice = 0; if (!ns) { - s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3118,8 +3119,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &(uint32_t){ htonl(INADDR_LOOPBACK) }; tref.tcp.splice = 1; - s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3144,8 +3145,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, tref.tcp.splice = 0; if (!ns) { - s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3159,8 +3160,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &in6addr_loopback; tref.tcp.splice = 1; - s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3193,7 +3194,7 @@ static int tcp_sock_init_ns(void *arg) if (!bitmap_isset(c->tcp.fwd_out.map, port)) continue; - tcp_sock_init(c, 1, AF_UNSPEC, NULL, port); + tcp_sock_init(c, 1, AF_UNSPEC, NULL, NULL, port); } return 0; @@ -3410,7 +3411,8 @@ static int tcp_port_rebind(void *arg) if ((a->c->ifi4 && tcp_sock_ns[port][V4] == -1) || (a->c->ifi6 && tcp_sock_ns[port][V6] == -1)) - tcp_sock_init(a->c, 1, AF_UNSPEC, NULL, port); + tcp_sock_init(a->c, 1, AF_UNSPEC, NULL, NULL, + port); } } else { for (port = 0; port < NUM_PORTS; port++) { @@ -3443,7 +3445,8 @@ static int tcp_port_rebind(void *arg) if ((a->c->ifi4 && tcp_sock_init_ext[port][V4] == -1) || (a->c->ifi6 && tcp_sock_init_ext[port][V6] == -1)) - tcp_sock_init(a->c, 0, AF_UNSPEC, NULL, port); + tcp_sock_init(a->c, 0, AF_UNSPEC, NULL, NULL, + port); } } diff --git a/tcp.h b/tcp.h index 2548d4d..7ba7ab7 100644 --- a/tcp.h +++ b/tcp.h @@ -21,7 +21,7 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events, int tcp_tap_handler(struct ctx *c, int af, const void *addr, const struct pool *p, const struct timespec *now); void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port); + const void *addr, const char *ifname, in_port_t port); int tcp_init(struct ctx *c); void tcp_timer(struct ctx *c, const struct timespec *ts); void tcp_defer_handler(struct ctx *c); diff --git a/udp.c b/udp.c index 5422fdd..cac9c65 100644 --- a/udp.c +++ b/udp.c @@ -1005,7 +1005,7 @@ int udp_tap_handler(struct ctx *c, int af, const void *addr, union udp_epoll_ref uref = { .udp.bound = 1, .udp.port = src }; - s = sock_l4(c, AF_INET, IPPROTO_UDP, NULL, src, + s = sock_l4(c, AF_INET, IPPROTO_UDP, NULL, NULL, src, uref.u32); if (s < 0) return p->count; @@ -1057,8 +1057,8 @@ int udp_tap_handler(struct ctx *c, int af, const void *addr, .udp.v6 = 1, .udp.port = src }; - s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, src, - uref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, NULL, + src, uref.u32); if (s < 0) return p->count; @@ -1111,10 +1111,11 @@ int udp_tap_handler(struct ctx *c, int af, const void *addr, * @ns: In pasta mode, if set, bind with loopback address in namespace * @af: Address family to select a specific IP version, or AF_UNSPEC * @addr: Pointer to address for binding, NULL if not configured + * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order */ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port) + const void *addr, const char *ifname, in_port_t port) { union udp_epoll_ref uref = { .udp.bound = 1 }; const void *bind_addr; @@ -1138,8 +1139,8 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, if (!ns) { uref.udp.splice = 0; - s = sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, port, - uref.u32); + s = sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); udp_tap_map[V4][uref.udp.port].sock = s; } @@ -1148,16 +1149,16 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &(uint32_t){ htonl(INADDR_LOOPBACK) }; uref.udp.splice = UDP_TO_NS; - sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } if (ns) { bind_addr = &(uint32_t){ htonl(INADDR_LOOPBACK) }; uref.udp.splice = UDP_TO_INIT; - sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } } @@ -1171,8 +1172,8 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, if (!ns) { uref.udp.splice = 0; - s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, port, - uref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); udp_tap_map[V6][uref.udp.port].sock = s; } @@ -1181,16 +1182,16 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &in6addr_loopback; uref.udp.splice = UDP_TO_NS; - sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } if (ns) { bind_addr = &in6addr_loopback; uref.udp.splice = UDP_TO_INIT; - sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } } } @@ -1213,7 +1214,7 @@ int udp_sock_init_ns(void *arg) if (!bitmap_isset(c->udp.fwd_out.f.map, dst)) continue; - udp_sock_init(c, 1, AF_UNSPEC, NULL, dst); + udp_sock_init(c, 1, AF_UNSPEC, NULL, NULL, dst); } return 0; diff --git a/udp.h b/udp.h index d14df0a..b4ee8b7 100644 --- a/udp.h +++ b/udp.h @@ -13,7 +13,7 @@ void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, int udp_tap_handler(struct ctx *c, int af, const void *addr, const struct pool *p, const struct timespec *now); void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port); + const void *addr, const char *ifname, in_port_t port); int udp_init(struct ctx *c); void udp_timer(struct ctx *c, const struct timespec *ts); void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s, diff --git a/util.c b/util.c index 5b1e08a..76057be 100644 --- a/util.c +++ b/util.c @@ -90,13 +90,15 @@ found: * @af: Address family, AF_INET or AF_INET6 * @proto: Protocol number * @bind_addr: Address for binding, NULL for any + * @ifname: Interface for binding, NULL for any * @port: Port, host order * @data: epoll reference portion for protocol handlers * * Return: newly created socket, -1 on error */ int sock_l4(const struct ctx *c, int af, uint8_t proto, - const void *bind_addr, uint16_t port, uint32_t data) + const void *bind_addr, const char *ifname, uint16_t port, + uint32_t data) { union epoll_ref ref = { .r.proto = proto, .r.p.data = data }; struct sockaddr_in addr4 = { @@ -163,6 +165,21 @@ int sock_l4(const struct ctx *c, int af, uint8_t proto, if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y))) debug("Failed to set SO_REUSEADDR on socket %i", fd); + if (ifname) { + /* Supported since kernel version 5.7, commit c427bfec18f2 + * ("net: core: enable SO_BINDTODEVICE for non-root users"). If + * it's unsupported, don't bind the socket at all, because the + * user might rely on this to filter incoming connections. + */ + if (setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, + ifname, strlen(ifname))) { + warn("Can't bind socket for %s port %u to %s, closing", + ip_proto_str[proto], port, ifname); + close(fd); + return -1; + } + } + if (bind(fd, sa, sl) < 0) { /* We'll fail to bind to low ports if we don't have enough * capabilities, and we'll fail to bind on already bound ports, diff --git a/util.h b/util.h index f9a8ec6..7dc3d18 100644 --- a/util.h +++ b/util.h @@ -194,7 +194,8 @@ __attribute__ ((weak)) int ffsl(long int i) { return __builtin_ffsl(i); } char *ipv6_l4hdr(const struct pool *p, int index, size_t offset, uint8_t *proto, size_t *dlen); int sock_l4(const struct ctx *c, int af, uint8_t proto, - const void *bind_addr, uint16_t port, uint32_t data); + const void *bind_addr, const char *ifname, uint16_t port, + uint32_t data); void sock_probe_mem(struct ctx *c); int timespec_diff_ms(const struct timespec *a, const struct timespec *b); void bitmap_set(uint8_t *map, int bit); -- 2.35.1

2 1

[PATCH v2 0/3] Fixes for spliced connections
by Stefano Brivio 13 Oct '22

13 Oct '22

The most pressing problem fixed here is that spliced connections with a remapped destination port would be attempted on the wrong side. This is an old issue as indicated by the Fixes: tag. Patch 1/3 restores sanity in comments before we attempt to fix the issue, patch 2/3 fixes the actual issue, and patch 3/3 introduces a minor rework on top to fix another issue, where if the user explicitly configures a loopback address in a port binding we would still create a non-spliced socket stealing the related connections. v2: - rewrite 2/3 with David's idea of using bit in epoll reference - rebase, and use IPV4_IS_LOOPBACK in tcp_conn_from_sock() too Stefano Brivio (3): tcp, tcp_splice: Adjust comments to current meaning of inbound and outbound tcp, tcp_splice: Fix port remapping for inbound, spliced connections Don't create 'tap' socket for ports that are bound to loopback only tcp.c | 192 +++++++++++++++++++++++++++++++-------------------- tcp.h | 2 + tcp_splice.c | 22 +++--- util.h | 3 + 4 files changed, 137 insertions(+), 82 deletions(-) -- 2.35.1

2 4

[PATCH] util.h: Add missing gcc pragma push before pragma pop
by Stefano Brivio 13 Oct '22

13 Oct '22

While building with clang: ./util.h:176:24: warning: pragma diagnostic pop could not pop, no matching push [-Wunknown-pragmas] Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- util.h | 1 + 1 file changed, 1 insertion(+) diff --git a/util.h b/util.h index b1eadfd..4d1b375 100644 --- a/util.h +++ b/util.h @@ -165,6 +165,7 @@ struct ctx; struct ipv6hdr { +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" #if __BYTE_ORDER == __BIG_ENDIAN uint8_t version:4, -- 2.35.1

2 1

[PATCH] log.h: Avoid unnecessary GNU extension for token pasting
by Stefano Brivio 13 Oct '22

13 Oct '22

clang says: ./log.h:23:18: warning: token pasting of ',' and __VA_ARGS__ is a GNU extension [-Wgnu-zero-variadic-macro-arguments] ...but we don't actually need token pasting there. Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.h b/log.h index f92394c..5abbfc5 100644 --- a/log.h +++ b/log.h @@ -20,7 +20,7 @@ void trace_init(int enable); #define trace(format, ...) \ do { \ if (log_trace) \ - debug(format, ##__VA_ARGS__); \ + debug(format, __VA_ARGS__); \ } while (0) void __openlog(const char *ident, int option, int facility); -- 2.35.1

2 1

[PATCH] icmp: Set sin6_scope_id for outbound ICMPv6 echo requests
by Stefano Brivio 13 Oct '22

13 Oct '22

If we ping a link-local address, we need to pass this to sendto(), as it will obviously fail with -EINVAL otherwise. If we ping other addresses, it's probably a good idea anyway to specify the configured outbound interface here. Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- icmp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/icmp.c b/icmp.c index 6bd87fd..21ea2d7 100644 --- a/icmp.c +++ b/icmp.c @@ -192,6 +192,7 @@ int icmp_tap_handler(const struct ctx *c, int af, const void *addr, struct sockaddr_in6 sa = { .sin6_family = AF_INET6, .sin6_addr = IN6ADDR_ANY_INIT, + .sin6_scope_id = c->ifi6, }; struct icmp6hdr *ih; int id, s; -- 2.35.1

2 1

[PATCH v3] conf, tcp, udp: Allow specification of interface to bind to
by Stefano Brivio 11 Oct '22

11 Oct '22

Since kernel version 5.7, commit c427bfec18f2 ("net: core: enable SO_BINDTODEVICE for non-root users"), we can bind sockets to interfaces, if they haven't been bound yet (as in bind()). Introduce an optional interface specification for forwarded ports, prefixed by %, that can be passed together with an address. Reported use case: running local services that use ports we want to have externally forwarded: https://github.com/containers/podman/issues/14425 Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- v3: - escape % characters in usage() formatting v2: - fix check on interface name length (spec - ifname, not ifname - buf) conf.c | 31 +++++++++++++++++++++---------- icmp.c | 4 ++-- passt.1 | 12 ++++++++++-- tcp.c | 27 +++++++++++++++------------ tcp.h | 2 +- udp.c | 35 ++++++++++++++++++----------------- udp.h | 2 +- util.c | 19 ++++++++++++++++++- util.h | 3 ++- 9 files changed, 88 insertions(+), 47 deletions(-) diff --git a/conf.c b/conf.c index 779371f..93ca0cd 100644 --- a/conf.c +++ b/conf.c @@ -180,8 +180,8 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, struct port_fwd *fwd) { char addr_buf[sizeof(struct in6_addr)] = { 0 }, *addr = addr_buf; + char buf[BUFSIZ], *spec, *ifname = NULL, *p; uint8_t exclude[PORT_BITMAP_SIZE] = { 0 }; - char buf[BUFSIZ], *spec, *p; sa_family_t af = AF_UNSPEC; bool exclude_only = true; @@ -209,9 +209,9 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, for (i = 0; i < PORT_EPHEMERAL_MIN; i++) { if (optname == 't') - tcp_sock_init(c, 0, AF_UNSPEC, NULL, i); + tcp_sock_init(c, 0, AF_UNSPEC, NULL, NULL, i); else if (optname == 'u') - udp_sock_init(c, 0, AF_UNSPEC, NULL, i); + udp_sock_init(c, 0, AF_UNSPEC, NULL, NULL, i); } return 0; @@ -231,6 +231,14 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, if (optname != 't' && optname != 'u') goto bad; + if ((ifname = strchr(buf, '%'))) { + if (spec - ifname >= IFNAMSIZ - 1) + goto bad; + + *ifname = 0; + ifname++; + } + if (inet_pton(AF_INET, buf, addr)) af = AF_INET; else if (inet_pton(AF_INET6, buf, addr)) @@ -278,9 +286,9 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, bitmap_set(fwd->map, i); if (optname == 't') - tcp_sock_init(c, 0, af, addr, i); + tcp_sock_init(c, 0, af, addr, NULL, i); else if (optname == 'u') - udp_sock_init(c, 0, af, addr, i); + udp_sock_init(c, 0, af, addr, NULL, i); } return 0; @@ -324,9 +332,9 @@ static int conf_ports(const struct ctx *c, char optname, const char *optarg, fwd->delta[i] = mapped_range.first - orig_range.first; if (optname == 't') - tcp_sock_init(c, 0, af, addr, i); + tcp_sock_init(c, 0, af, addr, ifname, i); else if (optname == 'u') - udp_sock_init(c, 0, af, addr, i); + udp_sock_init(c, 0, af, addr, ifname, i); } } while ((p = next_chunk(p, ','))); @@ -720,8 +728,9 @@ static void usage(const char *name) info( " 'all': forward all unbound, non-ephemeral ports"); info( " a comma-separated list, optionally ranged with '-'"); info( " and optional target ports after ':', with optional"); - info( " address specification suffixed by '/'. Ranges can be"); - info( " reduced by excluding ports or ranges prefixed by '~'"); + info( " address specification suffixed by '/' and optional"); + info( " interface prefixed by '%%'. Ranges can be reduced by"); + info( " excluding ports or ranges prefixed by '~'"); info( " Examples:"); info( " -t 22 Forward local port 22 to 22 on guest"); info( " -t 22:23 Forward local port 22 to 23 on guest"); @@ -740,6 +749,7 @@ static void usage(const char *name) exit(EXIT_FAILURE); pasta_opts: + info( " -t, --tcp-ports SPEC TCP port forwarding to namespace"); info( " can be specified multiple times"); info( " SPEC can be:"); @@ -747,7 +757,8 @@ pasta_opts: info( " 'auto': forward all ports currently bound in namespace"); info( " a comma-separated list, optionally ranged with '-'"); info( " and optional target ports after ':', with optional"); - info( " address specification suffixed by '/'. Examples:"); + info( " address specification suffixed by '/' and optional"); + info( " interface prefixed by '%%'. Examples:"); info( " -t 22 Forward local port 22 to port 22 in netns"); info( " -t 22:23 Forward local port 22 to port 23"); info( " -t 22,25 Forward ports 22, 25 to ports 22, 25"); diff --git a/icmp.c b/icmp.c index f02f89f..6bd87fd 100644 --- a/icmp.c +++ b/icmp.c @@ -169,7 +169,7 @@ int icmp_tap_handler(const struct ctx *c, int af, const void *addr, iref.icmp.id = id = ntohs(ih->un.echo.id); if ((s = icmp_id_map[V4][id].sock) <= 0) { - s = sock_l4(c, AF_INET, IPPROTO_ICMP, NULL, id, + s = sock_l4(c, AF_INET, IPPROTO_ICMP, NULL, NULL, id, iref.u32); if (s < 0) goto fail_sock; @@ -207,7 +207,7 @@ int icmp_tap_handler(const struct ctx *c, int af, const void *addr, iref.icmp.id = id = ntohs(ih->icmp6_identifier); if ((s = icmp_id_map[V6][id].sock) <= 0) { - s = sock_l4(c, AF_INET6, IPPROTO_ICMPV6, NULL, id, + s = sock_l4(c, AF_INET6, IPPROTO_ICMPV6, NULL, NULL, id, iref.u32); if (s < 0) goto fail_sock; diff --git a/passt.1 b/passt.1 index 555a50c..7d113f2 100644 --- a/passt.1 +++ b/passt.1 @@ -325,7 +325,8 @@ For low (< 1024) ports, see \fBNOTES\fR. .BR ports A comma-separated list of ports, optionally ranged with \fI-\fR, and, optionally, with target ports after \fI:\fR, if they differ. Specific addresses -can be bound as well, separated by \fI/\fR. Within given ranges, selected ports +can be bound as well, separated by \fI/\fR, and also, since Linux 5.7, limited +to specific interfaces, prefixed by \fI%\fR. Within given ranges, selected ports and ranges can be excluded by an additional specification prefixed by \fI~\fR. Specifying excluded ranges only implies that all other ports are forwarded. Examples: @@ -349,6 +350,9 @@ Forward local ports 22 to 80 to corresponding ports on the guest plus 10 -t 192.0.2.1/22 Forward local port 22, bound to 192.0.2.1, to port 22 on the guest .TP +-t 192.0.2.1%eth0/22 +Forward local port 22, bound to 192.0.2.1 and interface eth0, to port 22 +.TP -t 2000-5000,~3000-3010 Forward local ports 2000 to 5000, but not 3000 to 3010 .TP @@ -399,7 +403,8 @@ periodically derived (every second) from listening sockets reported by .BR ports A comma-separated list of ports, optionally ranged with \fI-\fR, and, optionally, with target ports after \fI:\fR, if they differ. Specific addresses -can be bound as well, separated by \fI/\fR. Within given ranges, selected ports +can be bound as well, separated by \fI/\fR, and also, since Linux 5.7, limited +to specific interfaces, prefixed by \fI%\fR. Within given ranges, selected ports and ranges can be excluded by an additional specification prefixed by \fI~\fR. Specifying excluded ranges only implies that all other ports are forwarded. Examples: @@ -424,6 +429,9 @@ namespace -t 192.0.2.1/22 Forward local port 22, bound to 192.0.2.1, to port 22 in the target namespace .TP +-t 192.0.2.1%eth0/22 +Forward local port 22, bound to 192.0.2.1 and interface eth0, to port 22 +.TP -t 2000-5000,~3000-3010 Forward local ports 2000 to 5000, but not 3000 to 3010 .TP diff --git a/tcp.c b/tcp.c index 830dc88..7e82589 100644 --- a/tcp.c +++ b/tcp.c @@ -3078,10 +3078,11 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events, * @ns: In pasta mode, if set, bind with loopback address in namespace * @af: Address family to select a specific IP version, or AF_UNSPEC * @addr: Pointer to address for binding, NULL if not configured + * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order */ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port) + const void *addr, const char *ifname, in_port_t port) { union tcp_epoll_ref tref = { .tcp.listen = 1 }; const void *bind_addr; @@ -3103,8 +3104,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, tref.tcp.splice = 0; if (!ns) { - s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3118,8 +3119,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &(uint32_t){ htonl(INADDR_LOOPBACK) }; tref.tcp.splice = 1; - s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3144,8 +3145,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, tref.tcp.splice = 0; if (!ns) { - s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3159,8 +3160,8 @@ void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &in6addr_loopback; tref.tcp.splice = 1; - s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, port, - tref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_TCP, bind_addr, ifname, + port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -3193,7 +3194,7 @@ static int tcp_sock_init_ns(void *arg) if (!bitmap_isset(c->tcp.fwd_out.map, port)) continue; - tcp_sock_init(c, 1, AF_UNSPEC, NULL, port); + tcp_sock_init(c, 1, AF_UNSPEC, NULL, NULL, port); } return 0; @@ -3410,7 +3411,8 @@ static int tcp_port_rebind(void *arg) if ((a->c->ifi4 && tcp_sock_ns[port][V4] == -1) || (a->c->ifi6 && tcp_sock_ns[port][V6] == -1)) - tcp_sock_init(a->c, 1, AF_UNSPEC, NULL, port); + tcp_sock_init(a->c, 1, AF_UNSPEC, NULL, NULL, + port); } } else { for (port = 0; port < NUM_PORTS; port++) { @@ -3443,7 +3445,8 @@ static int tcp_port_rebind(void *arg) if ((a->c->ifi4 && tcp_sock_init_ext[port][V4] == -1) || (a->c->ifi6 && tcp_sock_init_ext[port][V6] == -1)) - tcp_sock_init(a->c, 0, AF_UNSPEC, NULL, port); + tcp_sock_init(a->c, 0, AF_UNSPEC, NULL, NULL, + port); } } diff --git a/tcp.h b/tcp.h index 2548d4d..7ba7ab7 100644 --- a/tcp.h +++ b/tcp.h @@ -21,7 +21,7 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events, int tcp_tap_handler(struct ctx *c, int af, const void *addr, const struct pool *p, const struct timespec *now); void tcp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port); + const void *addr, const char *ifname, in_port_t port); int tcp_init(struct ctx *c); void tcp_timer(struct ctx *c, const struct timespec *ts); void tcp_defer_handler(struct ctx *c); diff --git a/udp.c b/udp.c index 5422fdd..cac9c65 100644 --- a/udp.c +++ b/udp.c @@ -1005,7 +1005,7 @@ int udp_tap_handler(struct ctx *c, int af, const void *addr, union udp_epoll_ref uref = { .udp.bound = 1, .udp.port = src }; - s = sock_l4(c, AF_INET, IPPROTO_UDP, NULL, src, + s = sock_l4(c, AF_INET, IPPROTO_UDP, NULL, NULL, src, uref.u32); if (s < 0) return p->count; @@ -1057,8 +1057,8 @@ int udp_tap_handler(struct ctx *c, int af, const void *addr, .udp.v6 = 1, .udp.port = src }; - s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, src, - uref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, NULL, + src, uref.u32); if (s < 0) return p->count; @@ -1111,10 +1111,11 @@ int udp_tap_handler(struct ctx *c, int af, const void *addr, * @ns: In pasta mode, if set, bind with loopback address in namespace * @af: Address family to select a specific IP version, or AF_UNSPEC * @addr: Pointer to address for binding, NULL if not configured + * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order */ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port) + const void *addr, const char *ifname, in_port_t port) { union udp_epoll_ref uref = { .udp.bound = 1 }; const void *bind_addr; @@ -1138,8 +1139,8 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, if (!ns) { uref.udp.splice = 0; - s = sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, port, - uref.u32); + s = sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); udp_tap_map[V4][uref.udp.port].sock = s; } @@ -1148,16 +1149,16 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &(uint32_t){ htonl(INADDR_LOOPBACK) }; uref.udp.splice = UDP_TO_NS; - sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } if (ns) { bind_addr = &(uint32_t){ htonl(INADDR_LOOPBACK) }; uref.udp.splice = UDP_TO_INIT; - sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } } @@ -1171,8 +1172,8 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, if (!ns) { uref.udp.splice = 0; - s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, port, - uref.u32); + s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); udp_tap_map[V6][uref.udp.port].sock = s; } @@ -1181,16 +1182,16 @@ void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, bind_addr = &in6addr_loopback; uref.udp.splice = UDP_TO_NS; - sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } if (ns) { bind_addr = &in6addr_loopback; uref.udp.splice = UDP_TO_INIT; - sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, port, - uref.u32); + sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, ifname, + port, uref.u32); } } } @@ -1213,7 +1214,7 @@ int udp_sock_init_ns(void *arg) if (!bitmap_isset(c->udp.fwd_out.f.map, dst)) continue; - udp_sock_init(c, 1, AF_UNSPEC, NULL, dst); + udp_sock_init(c, 1, AF_UNSPEC, NULL, NULL, dst); } return 0; diff --git a/udp.h b/udp.h index d14df0a..b4ee8b7 100644 --- a/udp.h +++ b/udp.h @@ -13,7 +13,7 @@ void udp_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, int udp_tap_handler(struct ctx *c, int af, const void *addr, const struct pool *p, const struct timespec *now); void udp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, in_port_t port); + const void *addr, const char *ifname, in_port_t port); int udp_init(struct ctx *c); void udp_timer(struct ctx *c, const struct timespec *ts); void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s, diff --git a/util.c b/util.c index 5b1e08a..76057be 100644 --- a/util.c +++ b/util.c @@ -90,13 +90,15 @@ found: * @af: Address family, AF_INET or AF_INET6 * @proto: Protocol number * @bind_addr: Address for binding, NULL for any + * @ifname: Interface for binding, NULL for any * @port: Port, host order * @data: epoll reference portion for protocol handlers * * Return: newly created socket, -1 on error */ int sock_l4(const struct ctx *c, int af, uint8_t proto, - const void *bind_addr, uint16_t port, uint32_t data) + const void *bind_addr, const char *ifname, uint16_t port, + uint32_t data) { union epoll_ref ref = { .r.proto = proto, .r.p.data = data }; struct sockaddr_in addr4 = { @@ -163,6 +165,21 @@ int sock_l4(const struct ctx *c, int af, uint8_t proto, if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y))) debug("Failed to set SO_REUSEADDR on socket %i", fd); + if (ifname) { + /* Supported since kernel version 5.7, commit c427bfec18f2 + * ("net: core: enable SO_BINDTODEVICE for non-root users"). If + * it's unsupported, don't bind the socket at all, because the + * user might rely on this to filter incoming connections. + */ + if (setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, + ifname, strlen(ifname))) { + warn("Can't bind socket for %s port %u to %s, closing", + ip_proto_str[proto], port, ifname); + close(fd); + return -1; + } + } + if (bind(fd, sa, sl) < 0) { /* We'll fail to bind to low ports if we don't have enough * capabilities, and we'll fail to bind on already bound ports, diff --git a/util.h b/util.h index f9a8ec6..7dc3d18 100644 --- a/util.h +++ b/util.h @@ -194,7 +194,8 @@ __attribute__ ((weak)) int ffsl(long int i) { return __builtin_ffsl(i); } char *ipv6_l4hdr(const struct pool *p, int index, size_t offset, uint8_t *proto, size_t *dlen); int sock_l4(const struct ctx *c, int af, uint8_t proto, - const void *bind_addr, uint16_t port, uint32_t data); + const void *bind_addr, const char *ifname, uint16_t port, + uint32_t data); void sock_probe_mem(struct ctx *c); int timespec_diff_ms(const struct timespec *a, const struct timespec *b); void bitmap_set(uint8_t *map, int bit); -- 2.35.1

2 2

[PATCH 0/3] Fixes for spliced connections
by Stefano Brivio 11 Oct '22

11 Oct '22

The most pressing problem fixed here is that spliced connections with a remapped destination port would be attempted on the wrong side. This is an old issue as indicated by the Fixes: tag. Patch 1/3 restores sanity in comments before we attempt to fix the issue, patch 2/3 fixes the actual issue, and patch 3/3 introduces a minor rework on top to fix another issue, where if the user explicitly configures a loopback address in a port binding we would still create a non-spliced socket stealing the related connections. Stefano Brivio (3): tcp, tcp_splice: Adjust comments to current meaning of inbound and outbound tcp, tcp_splice: Fix port remapping for inbound, spliced connections tcp: Don't create 'tap' socket for ports that are bound to loopback only tcp.c | 187 ++++++++++++++++++++++++++++++++------------------- tcp_splice.c | 20 ++++-- util.h | 3 + 3 files changed, 134 insertions(+), 76 deletions(-) -- 2.35.1

2 6

[PATCH] conf: Drop excess colons in usage for DHCP and DNS options
by Stefano Brivio 11 Oct '22

11 Oct '22

Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- conf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf.c b/conf.c index c2634fc..b1b7c63 100644 --- a/conf.c +++ b/conf.c @@ -693,14 +693,14 @@ static void usage(const char *name) info( " default: use search list from /etc/resolv.conf"); if (strstr(name, "pasta")) - info(" --dhcp-dns: \tPass DNS list via DHCP/DHCPv6/NDP"); + info(" --dhcp-dns \tPass DNS list via DHCP/DHCPv6/NDP"); else - info(" --no-dhcp-dns: No DNS list in DHCP/DHCPv6/NDP"); + info(" --no-dhcp-dns No DNS list in DHCP/DHCPv6/NDP"); if (strstr(name, "pasta")) - info(" --dhcp-search: Pass list via DHCP/DHCPv6/NDP"); + info(" --dhcp-search Pass list via DHCP/DHCPv6/NDP"); else - info(" --no-dhcp-search: No list in DHCP/DHCPv6/NDP"); + info(" --no-dhcp-search No list in DHCP/DHCPv6/NDP"); info( " --dns-forward ADDR Forward DNS queries sent to ADDR"); info( " can be specified zero to two times (for IPv4 and IPv6)"); -- 2.35.1

2 1

[PATCH] netlink: Disable duplicate address detection for configured IPv6 address
by Stefano Brivio 11 Oct '22

11 Oct '22

With default options, when we pass --config-net, the IPv6 address is actually going to be recycled from the init namespace, so it is in fact duplicated, but duplicate address detection has no way to find out. With a different configured address, that's not the case, but anyway duplicate address detection will be unable to see this. In both cases, we're wasting time for nothing. Pass the IFA_F_NODAD flag as we configure globally scoped IPv6 addresses via netlink. Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com> --- netlink.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/netlink.c b/netlink.c index 9719e91..6e5a96b 100644 --- a/netlink.c +++ b/netlink.c @@ -343,6 +343,9 @@ void nl_addr(int ns, unsigned int ifi, sa_family_t af, if (af == AF_INET6) { size_t rta_len = RTA_LENGTH(sizeof(req.set.a6.l)); + /* By default, strictly speaking, it's duplicated */ + req.ifa.ifa_flags = IFA_F_NODAD; + req.nlh.nlmsg_len = offsetof(struct req_t, set.a6) + sizeof(req.set.a6); -- 2.35.1

2 1

2025

2024

2023

2022

2021

dev