From 930bdfbc7796e4f3a9661198bc924869c8620256 Mon Sep 17 00:00:00 2001 From: akshitguptaa Date: Tue, 4 Aug 2026 02:48:56 +0530 Subject: [PATCH] port/builtin: switch from iptables to nft for transparent routing Signed-off-by: akshitguptaa --- .github/workflows/main.yaml | 10 +++ Dockerfile | 12 ++-- README.md | 1 + cmd/rootlesskit/main.go | 14 +++- docs/port.md | 2 +- pkg/api/api.go | 9 +-- pkg/api/openapi.yaml | 7 +- pkg/port/builtin/builtin.go | 4 +- pkg/port/builtin/builtin_test.go | 41 ++++++++++- pkg/port/builtin/child/child.go | 113 +++++++++++++++++++++++++----- pkg/port/builtin/opaque/opaque.go | 7 +- pkg/port/builtin/parent/parent.go | 50 ++++++++----- 12 files changed, 220 insertions(+), 50 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e1f68de4..b7222d35 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -11,6 +11,16 @@ jobs: run: DOCKER_BUILDKIT=1 docker build -t rootlesskit:test-unit --target test-unit . - name: "Unit test" run: docker run --rm --privileged rootlesskit:test-unit + test-unit-iptables-fallback: + name: "Unit test (source-ip-transparent iptables fallback, no nft)" + runs-on: ubuntu-24.04 + steps: + - name: "Check out" + uses: actions/checkout@v7 + - name: "Build unit test image without nft" + run: DOCKER_BUILDKIT=1 docker build -t rootlesskit:test-unit-iptables-fallback --target test-unit --build-arg TEST_UNIT_APT_EXTRA=iptables . + - name: "Unit test" + run: docker run --rm --privileged rootlesskit:test-unit-iptables-fallback test-cross: name: "Cross compilation test" runs-on: ubuntu-24.04 diff --git a/Dockerfile b/Dockerfile index 100ba8a5..b0f3d18f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,11 @@ COPY --from=cross /go/src/github.com/rootless-containers/rootlesskit/_artifact/* # `go test -race` requires non-Alpine FROM golang:${GO_VERSION} AS test-unit -# iptables: used for source-ip-transparent -RUN apt-get update && apt-get install -y git iproute2 netcat-openbsd iptables +# iptables, nftables: used for source-ip-transparent (nft preferred, iptables as fallback). +# TEST_UNIT_APT_EXTRA can be overridden to "iptables" only, to exercise the +# fallback path in CI when nft isn't available. +ARG TEST_UNIT_APT_EXTRA="iptables nftables" +RUN apt-get update && apt-get install -y git iproute2 netcat-openbsd $TEST_UNIT_APT_EXTRA ADD . /go/src/github.com/rootless-containers/rootlesskit WORKDIR /go/src/github.com/rootless-containers/rootlesskit RUN go mod verify && go vet ./... @@ -65,8 +68,9 @@ FROM ubuntu:${UBUNTU_VERSION} AS test-integration # libcap2-bin and curl: used by the RUN instructions in this Dockerfile. # bind9-dnsutils: for `nslookup` command used by integration-net.sh # systemd and uuid-runtime: for systemd-socket-activate used by integration-systemd-socket.sh -# iptables: for source-ip-transparent. Also for Docker. -RUN apt-get update && apt-get install -y iproute2 liblxc-common lxc-utils iperf3 busybox sudo libcap2-bin curl bind9-dnsutils systemd uuid-runtime iptables +# iptables: for Docker (dockerd-rootless itself still uses iptables). +# nftables: for source-ip-transparent (rootlesskit's own builtin port driver). +RUN apt-get update && apt-get install -y iproute2 liblxc-common lxc-utils iperf3 busybox sudo libcap2-bin curl bind9-dnsutils systemd uuid-runtime iptables nftables COPY --from=idmap /usr/bin/newuidmap /usr/bin/newuidmap COPY --from=idmap /usr/bin/newgidmap /usr/bin/newgidmap RUN /sbin/setcap cap_setuid+eip /usr/bin/newuidmap && \ diff --git a/README.md b/README.md index 084776c5..f1c74519 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,7 @@ OPTIONS: --port-driver value port driver for non-host network. [none, implicit (for pasta), builtin, slirp4netns, gvisor-tap-vsock(experimental)] (default: "none") --publish value, -p value [ --publish value, -p value ] publish ports. e.g. "127.0.0.1:8080:80/tcp" --source-ip-transparent preserve real client source IP using IP_TRANSPARENT (builtin port driver, TCP only) (default: true) + --source-ip-transparent-backend value firewall backend for --source-ip-transparent (builtin port driver) [auto, nft, iptables] (default: "auto") Process: --pidns create a PID namespace (default: false) diff --git a/cmd/rootlesskit/main.go b/cmd/rootlesskit/main.go index 1d83cb3d..e1eb3621 100644 --- a/cmd/rootlesskit/main.go +++ b/cmd/rootlesskit/main.go @@ -209,6 +209,11 @@ See https://rootlesscontaine.rs/getting-started/common/ . Usage: "preserve real client source IP using IP_TRANSPARENT (builtin port driver, TCP only)", Value: true, }, CategoryPort), + Categorize(&cli.StringFlag{ + Name: "source-ip-transparent-backend", + Usage: "firewall backend for --source-ip-transparent (builtin port driver) [auto, nft, iptables]", + Value: "auto", + }, CategoryPort), Categorize(&cli.BoolFlag{ Name: "pidns", Usage: "create a PID namespace", @@ -625,7 +630,14 @@ func createParentOpt(clicontext *cli.Context) (parent.Opt, error) { if opt.NetworkDriver == nil { return opt, errors.New("port driver requires non-host network") } - opt.PortDriver, err = builtin.NewParentDriver(&logrusDebugWriter{label: "port/builtin"}, opt.StateDir, clicontext.Bool("source-ip-transparent")) + sourceIPTransparentBackend := clicontext.String("source-ip-transparent-backend") + switch sourceIPTransparentBackend { + case "auto", "nft", "iptables": + // OK + default: + return opt, fmt.Errorf("unknown source-ip-transparent-backend: %s", sourceIPTransparentBackend) + } + opt.PortDriver, err = builtin.NewParentDriver(&logrusDebugWriter{label: "port/builtin"}, opt.StateDir, clicontext.Bool("source-ip-transparent"), sourceIPTransparentBackend) if err != nil { return opt, err } diff --git a/docs/port.md b/docs/port.md index 4097237f..c845be7b 100644 --- a/docs/port.md +++ b/docs/port.md @@ -7,7 +7,7 @@ The default value is `none` (do not expose ports). | `--port-driver` | Throughput | Source IP | Notes |----------------------|-------------|----------|------- | `slirp4netns` | 8.03 Gbps | Propagated | -| `builtin` | 29.9 Gbps | Propagated for TCP (since v3.0) | Source IP propagation (`--source-ip-transparent`) applies to TCP only; UDP is not propagated. In the case of Rootless Docker, userland-proxy has to be disabled for propagating the source IP. +| `builtin` | 29.9 Gbps | Propagated for TCP (since v3.0) | Source IP propagation (`--source-ip-transparent`) applies to TCP only; UDP is not propagated. In the case of Rootless Docker, userland-proxy has to be disabled for propagating the source IP. The underlying firewall rules use `nft`, falling back to `iptables` if `nft` is unavailable; `--source-ip-transparent-backend` can be used to force one or the other. | `implicit` | 37.6 Gbps | Propagated | Requires `pasta` network | `gvisor-tap-vsock` (Experimental) | 3.83 Gbps | Not propagated | Throughput is currently limited; see issue link below for improvement ideas. diff --git a/pkg/api/api.go b/pkg/api/api.go index b82e633c..5877e549 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -5,7 +5,7 @@ import "net" const ( // Version of the REST API, not implementation version. // See openapi.yaml for the definition. - Version = "1.1.2" + Version = "1.1.3" ) // Info is the structure returned by `GET /info` @@ -29,7 +29,8 @@ type NetworkDriverInfo struct { // PortDriverInfo in Info type PortDriverInfo struct { - Driver string `json:"driver"` - Protos []string `json:"protos"` - DisallowLoopbackChildIP bool `json:"disallowLoopbackChildIP,omitempty"` // since API v1.1.1 + Driver string `json:"driver"` + Protos []string `json:"protos"` + DisallowLoopbackChildIP bool `json:"disallowLoopbackChildIP,omitempty"` // since API v1.1.1 + Extra map[string]string `json:"extra,omitempty"` // since API v1.1.3, driver-specific details, e.g. {"sourceIPTransparentBackend": "nft"} for the builtin driver } diff --git a/pkg/api/openapi.yaml b/pkg/api/openapi.yaml index 5aef6790..d36f3d95 100644 --- a/pkg/api/openapi.yaml +++ b/pkg/api/openapi.yaml @@ -1,7 +1,7 @@ # When you made a change to this YAML, please validate with https://editor.swagger.io openapi: 3.0.3 info: - version: 1.1.2 + version: 1.1.3 title: RootlessKit API servers: - url: 'http://rootlesskit/v1' @@ -172,3 +172,8 @@ components: disallowLoopbackChildIP: type: boolean description: "If this field is set to true, loopback IP such as 127.0.0.1 cannot be specified as a child IP" + extra: + type: object + description: "Driver-specific details, e.g. {\"sourceIPTransparentBackend\": \"nft\"} for the builtin driver" + additionalProperties: + type: string diff --git a/pkg/port/builtin/builtin.go b/pkg/port/builtin/builtin.go index c1c15554..6020a7b7 100644 --- a/pkg/port/builtin/builtin.go +++ b/pkg/port/builtin/builtin.go @@ -9,8 +9,8 @@ import ( ) var ( - NewParentDriver func(logWriter io.Writer, stateDir string, sourceIPTransparent bool) (port.ParentDriver, error) = parent.NewDriver - NewChildDriver func(logWriter io.Writer) port.ChildDriver = child.NewDriver + NewParentDriver func(logWriter io.Writer, stateDir string, sourceIPTransparent bool, sourceIPTransparentBackend string) (port.ParentDriver, error) = parent.NewDriver + NewChildDriver func(logWriter io.Writer) port.ChildDriver = child.NewDriver ) // Available indicates whether this port driver is compiled in (used for generating help text) diff --git a/pkg/port/builtin/builtin_test.go b/pkg/port/builtin/builtin_test.go index 2228ab23..f3e475d7 100644 --- a/pkg/port/builtin/builtin_test.go +++ b/pkg/port/builtin/builtin_test.go @@ -1,7 +1,9 @@ package builtin import ( + "context" "os" + "os/exec" "testing" "github.com/rootless-containers/rootlesskit/v3/pkg/port" @@ -21,7 +23,7 @@ func TestBuiltIn(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpDir) - d, err := NewParentDriver(os.Stderr, tmpDir, true) + d, err := NewParentDriver(os.Stderr, tmpDir, true, "auto") if err != nil { t.Fatal(err) } @@ -32,3 +34,40 @@ func TestBuiltIn(t *testing.T) { testsuite.RunTCPTransparent(t, pf) testsuite.RunUDPTransparent(t, pf) } + +// TestSourceIPTransparentBackend exercises an explicit +// --source-ip-transparent-backend selection end to end, and checks that the +// choice is reported back via PortDriverInfo.Extra. +func TestSourceIPTransparentBackend(t *testing.T) { + for _, backend := range []string{"nft", "iptables"} { + t.Run(backend, func(t *testing.T) { + if backend == "nft" { + ensureNFT(t) + } + tmpDir, err := os.MkdirTemp("", "test-builtin-backend") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + d, err := NewParentDriver(os.Stderr, tmpDir, true, backend) + if err != nil { + t.Fatal(err) + } + info, err := d.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := info.Extra["sourceIPTransparentBackend"]; got != backend { + t.Fatalf("expected PortDriverInfo.Extra[sourceIPTransparentBackend]=%q, got %q", backend, got) + } + testsuite.RunTCPTransparent(t, func() port.ParentDriver { return d }) + }) + } +} + +func ensureNFT(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("nft"); err != nil { + t.Skipf("nft not found: %v", err) + } +} diff --git a/pkg/port/builtin/child/child.go b/pkg/port/builtin/child/child.go index 09cce0b9..bb6928a1 100644 --- a/pkg/port/builtin/child/child.go +++ b/pkg/port/builtin/child/child.go @@ -29,15 +29,20 @@ func NewDriver(logWriter io.Writer) port.ChildDriver { } type childDriver struct { - logWriter io.Writer - sourceIPTransparent bool - routingSetup sync.Once - routingReady bool - routingWarn sync.Once + logWriter io.Writer + sourceIPTransparent bool + sourceIPTransparentBackend string // "auto" (default), "nft", or "iptables" + routingSetup sync.Once + routingReady bool + routingWarn sync.Once } func (d *childDriver) RunChildDriver(opaque map[string]string, quit <-chan struct{}, detachedNetNSPath string) error { d.sourceIPTransparent = opaque[opaquepkg.SourceIPTransparent] == "true" + d.sourceIPTransparentBackend = opaque[opaquepkg.SourceIPTransparentBackend] + if d.sourceIPTransparentBackend == "" { + d.sourceIPTransparentBackend = "auto" + } socketPath := opaque[opaquepkg.SocketPath] if socketPath == "" { return errors.New("socket path not set") @@ -207,6 +212,9 @@ fallback: // setupTransparentRouting sets up policy routing so that response packets // destined to transparent-bound source IPs are delivered locally. +// The firewall rules are implemented via nft, falling back to iptables if +// nft isn't available on the host (see setupTransparentRoutingNFT and +// setupTransparentRoutingIPTables). // // Transparent sockets (IP_TRANSPARENT) bind to non-local addresses (the real // client IP). Response packets to these addresses must be routed locally instead @@ -224,17 +232,8 @@ fallback: // SYN-ACK is then routed via the fwmark table (local delivery) instead of // the default route (TAP), allowing it to reach the transparent socket. func (d *childDriver) setupTransparentRouting() bool { - // Check that iptables is available before proceeding. - if _, err := exec.LookPath("iptables"); err != nil { - fmt.Fprintf(d.logWriter, "source IP transparent: iptables not found, disabling: %v\n", err) - return false - } - // Verify the connmark module is usable (kernel module might not be loaded). - if out, err := exec.Command("iptables", "-t", "mangle", "-L", "-n").CombinedOutput(); err != nil { - fmt.Fprintf(d.logWriter, "source IP transparent: iptables mangle table not available, disabling: %v: %s\n", err, out) - return false - } - cmds := [][]string{ + // Common prep, independent of the firewall backend used below. + prepCmds := [][]string{ // Table 100: treat all addresses as local (for delivery to transparent sockets) {"ip", "route", "add", "local", "default", "dev", "lo", "table", "100"}, {"ip", "-6", "route", "add", "local", "default", "dev", "lo", "table", "100"}, @@ -244,6 +243,86 @@ func (d *childDriver) setupTransparentRouting() bool { // Inherit fwmark from SYN to accepted socket (needed for userspace proxies // like docker-proxy, so that SYN-ACK routing uses table 100) {"sysctl", "-w", "net.ipv4.tcp_fwmark_accept=1"}, + } + for _, args := range prepCmds { + if out, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil { + fmt.Fprintf(d.logWriter, "source IP transparent routing setup: %v: %s\n", err, out) + } + } + switch d.sourceIPTransparentBackend { + case "nft": + if d.setupTransparentRoutingNFT() { + return true + } + fmt.Fprintf(d.logWriter, "source IP transparent: nft backend was explicitly requested but is unavailable\n") + return false + case "iptables": + return d.setupTransparentRoutingIPTables() + default: // "auto" + if d.setupTransparentRoutingNFT() { + return true + } + fmt.Fprintf(d.logWriter, "source IP transparent: nft unavailable, falling back to iptables\n") + return d.setupTransparentRoutingIPTables() + } +} + +// setupTransparentRoutingNFT implements setupTransparentRouting using nft. +// The "inet" family covers both IPv4 and IPv6 in a single ruleset. +func (d *childDriver) setupTransparentRoutingNFT() bool { + // Check that nft is available before proceeding. + if _, err := exec.LookPath("nft"); err != nil { + fmt.Fprintf(d.logWriter, "source IP transparent (nft): nft not found, disabling: %v\n", err) + return false + } + // Verify nftables is usable (the nf_tables kernel module might not be loaded). + if out, err := exec.Command("nft", "list", "tables").CombinedOutput(); err != nil { + fmt.Fprintf(d.logWriter, "source IP transparent (nft): nft not available, disabling: %v: %s\n", err, out) + return false + } + const nftTable = "rootlesskit_transparent" + cmds := [][]string{ + // Create a single nftables table in the "inet" family, which covers both + // IPv4 and IPv6, replacing the separate iptables/ip6tables tables used below. + {"nft", "add", "table", "inet", nftTable}, + // Hook into OUTPUT and PREROUTING at the "mangle" priority, matching where + // the equivalent rules live in the iptables mangle table. + {"nft", "add", "chain", "inet", nftTable, "output", + "{", "type", "filter", "hook", "output", "priority", "mangle", ";", "}"}, + {"nft", "add", "chain", "inet", nftTable, "prerouting", + "{", "type", "filter", "hook", "prerouting", "priority", "mangle", ";", "}"}, + // In OUTPUT: tag transparent connections (non-local source) with a connection + // mark. Equivalent to iptables: -m addrtype ! --src-type LOCAL -j CONNMARK --set-mark 100 + {"nft", "add", "rule", "inet", nftTable, "output", + "meta", "l4proto", "tcp", "fib", "saddr", "type", "!=", "local", "ct", "mark", "set", "100"}, + // In PREROUTING: restore the connmark to the packet mark for routing. + // Equivalent to iptables: -m connmark --mark 100 -j MARK --set-mark 100 + {"nft", "add", "rule", "inet", nftTable, "prerouting", + "meta", "l4proto", "tcp", "ct", "mark", "100", "meta", "mark", "set", "100"}, + } + for _, args := range cmds { + if out, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil { + fmt.Fprintf(d.logWriter, "source IP transparent (nft) routing setup: %v: %s\n", err, out) + } + } + return true +} + +// setupTransparentRoutingIPTables implements setupTransparentRouting using +// iptables/ip6tables, kept as a fallback for hosts where nft is unavailable +// and for environments that still expect iptables specifically. +func (d *childDriver) setupTransparentRoutingIPTables() bool { + // Check that iptables is available before proceeding. + if _, err := exec.LookPath("iptables"); err != nil { + fmt.Fprintf(d.logWriter, "source IP transparent (iptables): iptables not found, disabling: %v\n", err) + return false + } + // Verify the connmark module is usable (kernel module might not be loaded). + if out, err := exec.Command("iptables", "-t", "mangle", "-L", "-n").CombinedOutput(); err != nil { + fmt.Fprintf(d.logWriter, "source IP transparent (iptables): mangle table not available, disabling: %v: %s\n", err, out) + return false + } + cmds := [][]string{ // In OUTPUT: tag transparent connections (non-local source) with CONNMARK {"iptables", "-t", "mangle", "-A", "OUTPUT", "-p", "tcp", "-m", "addrtype", "!", "--src-type", "LOCAL", "-j", "CONNMARK", "--set-mark", "100"}, {"ip6tables", "-t", "mangle", "-A", "OUTPUT", "-p", "tcp", "-m", "addrtype", "!", "--src-type", "LOCAL", "-j", "CONNMARK", "--set-mark", "100"}, @@ -253,7 +332,7 @@ func (d *childDriver) setupTransparentRouting() bool { } for _, args := range cmds { if out, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil { - fmt.Fprintf(d.logWriter, "source IP transparent routing setup: %v: %s\n", err, out) + fmt.Fprintf(d.logWriter, "source IP transparent (iptables) routing setup: %v: %s\n", err, out) } } return true diff --git a/pkg/port/builtin/opaque/opaque.go b/pkg/port/builtin/opaque/opaque.go index 1a21352f..6811ca4f 100644 --- a/pkg/port/builtin/opaque/opaque.go +++ b/pkg/port/builtin/opaque/opaque.go @@ -1,7 +1,8 @@ package opaque const ( - SocketPath = "builtin.socketpath" - ChildReadyPipePath = "builtin.readypipepath" - SourceIPTransparent = "builtin.source-ip-transparent" + SocketPath = "builtin.socketpath" + ChildReadyPipePath = "builtin.readypipepath" + SourceIPTransparent = "builtin.source-ip-transparent" + SourceIPTransparentBackend = "builtin.source-ip-transparent-backend" ) diff --git a/pkg/port/builtin/parent/parent.go b/pkg/port/builtin/parent/parent.go index 4abb767b..67a4fcbc 100644 --- a/pkg/port/builtin/parent/parent.go +++ b/pkg/port/builtin/parent/parent.go @@ -24,7 +24,8 @@ import ( ) // NewDriver for builtin driver. -func NewDriver(logWriter io.Writer, stateDir string, sourceIPTransparent bool) (port.ParentDriver, error) { +// sourceIPTransparentBackend is one of "auto" (default), "nft", or "iptables". +func NewDriver(logWriter io.Writer, stateDir string, sourceIPTransparent bool, sourceIPTransparentBackend string) (port.ParentDriver, error) { // TODO: consider using socketpair FD instead of socket file socketPath := filepath.Join(stateDir, ".bp.sock") childReadyPipePath := filepath.Join(stateDir, ".bp-ready.pipe") @@ -35,27 +36,32 @@ func NewDriver(logWriter io.Writer, stateDir string, sourceIPTransparent bool) ( if err := syscall.Mkfifo(childReadyPipePath, 0600); err != nil { return nil, fmt.Errorf("cannot mkfifo %s: %w", childReadyPipePath, err) } + if sourceIPTransparentBackend == "" { + sourceIPTransparentBackend = "auto" + } d := driver{ - logWriter: logWriter, - socketPath: socketPath, - childReadyPipePath: childReadyPipePath, - sourceIPTransparent: sourceIPTransparent, - ports: make(map[int]*port.Status, 0), - stoppers: make(map[int]func(context.Context) error, 0), - nextID: 1, + logWriter: logWriter, + socketPath: socketPath, + childReadyPipePath: childReadyPipePath, + sourceIPTransparent: sourceIPTransparent, + sourceIPTransparentBackend: sourceIPTransparentBackend, + ports: make(map[int]*port.Status, 0), + stoppers: make(map[int]func(context.Context) error, 0), + nextID: 1, } return &d, nil } type driver struct { - logWriter io.Writer - socketPath string - childReadyPipePath string - sourceIPTransparent bool - mu sync.Mutex - ports map[int]*port.Status - stoppers map[int]func(context.Context) error - nextID int + logWriter io.Writer + socketPath string + childReadyPipePath string + sourceIPTransparent bool + sourceIPTransparentBackend string + mu sync.Mutex + ports map[int]*port.Status + stoppers map[int]func(context.Context) error + nextID int } func (d *driver) Info(ctx context.Context) (*api.PortDriverInfo, error) { @@ -64,6 +70,17 @@ func (d *driver) Info(ctx context.Context) (*api.PortDriverInfo, error) { Protos: []string{"tcp", "tcp4", "tcp6", "udp", "udp4", "udp6"}, DisallowLoopbackChildIP: false, } + if d.sourceIPTransparent { + // This reflects the configured backend selection (the + // --source-ip-transparent-backend value), not which backend the + // child process actually ended up using at runtime: the child only + // picks nft vs. iptables lazily, on the first connection that needs + // source IP transparency, so the real outcome isn't known to the + // parent at Info() time. + info.Extra = map[string]string{ + "sourceIPTransparentBackend": d.sourceIPTransparentBackend, + } + } return info, nil } @@ -74,6 +91,7 @@ func (d *driver) OpaqueForChild() map[string]string { } if d.sourceIPTransparent { m[opaque.SourceIPTransparent] = "true" + m[opaque.SourceIPTransparentBackend] = d.sourceIPTransparentBackend } return m }