Support rootless containerd worker

Signed-off-by: Kohei Tokunaga <[email protected]>

.github/workflows/build.yml

@@ -69,6 +69,7 @@ jobs:
           - ./frontend/dockerfile
         worker:
           - containerd
+          - containerd-rootless
           - containerd-1.5
           - containerd-1.4
           - containerd-snapshotter-stargz

cmd/buildkitd/config/config.go

@@ -99,6 +99,8 @@ type ContainerdConfig struct {
 	ApparmorProfile string `toml:"apparmor-profile"`
 
 	MaxParallelism int `toml:"max-parallelism"`
+
+	Rootless bool `toml:"rootless"`
 }
 
 type GCPolicy struct {

cmd/buildkitd/main_containerd_worker.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	ctd "github.com/containerd/containerd"
+	"github.com/containerd/containerd/pkg/userns"
 	"github.com/moby/buildkit/cmd/buildkitd/config"
 	"github.com/moby/buildkit/util/network/cniprovider"
 	"github.com/moby/buildkit/util/network/netproviders"
@@ -99,6 +100,19 @@ func init() {
 			Usage: "set the name of the apparmor profile applied to containers",
 		},
 	}
+	n := "containerd-worker-rootless"
+	u := "enable rootless mode"
+	if userns.RunningInUserNS() {
+		flags = append(flags, cli.BoolTFlag{
+			Name:  n,
+			Usage: u,
+		})
+	} else {
+		flags = append(flags, cli.BoolFlag{
+			Name:  n,
+			Usage: u,
+		})
+	}
 
 	if defaultConf.Workers.Containerd.GC == nil || *defaultConf.Workers.Containerd.GC {
 		flags = append(flags, cli.BoolTFlag{
@@ -147,13 +161,14 @@ func applyContainerdFlags(c *cli.Context, cfg *config.Config) error {
 		cfg.Workers.Containerd.Enabled = boolOrAuto
 	}
 
-	// GlobalBool works for BoolT as well
-	rootless := c.GlobalBool("rootless")
-	if rootless {
-		logrus.Warn("rootless mode is not supported for containerd workers. disabling containerd worker.")
-		b := false
-		cfg.Workers.Containerd.Enabled = &b
-		return nil
+	if c.GlobalIsSet("rootless") || c.GlobalBool("rootless") {
+		cfg.Workers.Containerd.Rootless = c.GlobalBool("rootless")
+	}
+	if c.GlobalIsSet("containerd-worker-rootless") {
+		if !userns.RunningInUserNS() || os.Geteuid() > 0 {
+			return errors.New("rootless mode requires to be executed as the mapped root in a user namespace; you may use RootlessKit for setting up the namespace")
+		}
+		cfg.Workers.Containerd.Rootless = c.GlobalBool("containerd-worker-rootless")
 	}
 
 	labels, err := attrMap(c.GlobalStringSlice("containerd-worker-labels"))
@@ -217,6 +232,13 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 		return nil, nil
 	}
 
+	if cfg.Rootless {
+		logrus.Debugf("running in rootless mode")
+		if common.config.Workers.Containerd.NetworkConfig.Mode == "auto" {
+			common.config.Workers.Containerd.NetworkConfig.Mode = "host"
+		}
+	}
+
 	dns := getDNSConfig(common.config.DNS)
 
 	nc := netproviders.Opt{
@@ -237,7 +259,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 	if cfg.Snapshotter != "" {
 		snapshotter = cfg.Snapshotter
 	}
-	opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, parallelismSem, common.traceSocket, ctd.WithTimeout(60*time.Second))
+	opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Rootless, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, parallelismSem, common.traceSocket, ctd.WithTimeout(60*time.Second))
 	if err != nil {
 		return nil, err
 	}

docs/rootless.md

@@ -24,10 +24,8 @@ You may have to disable SELinux, or run BuildKit with `--oci-worker-snapshotter=
   On kernel >= 4.18, the `fuse-overlayfs` snapshotter is used instead of `overlayfs`.
   On kernel < 4.18, the `native` snapshotter is used.
 * Network mode is always set to `network.host`.
-* No support for `containerd` worker.
-  ("worker" here is a BuildKit term, not a Kubernetes term. Running rootless BuildKit in containerd is fully supported.)
 
-## Running BuildKit in Rootless mode
+## Running BuildKit in Rootless mode (OCI worker)
 
 [RootlessKit](https://github.com/rootless-containers/rootlesskit/) needs to be installed.
 
@@ -44,6 +42,22 @@ To isolate BuildKit daemon's network namespace from the host (recommended):
 $ rootlesskit --net=slirp4netns --copy-up=/etc --disable-host-loopback buildkitd
 ```
 
+## Running BuildKit in Rootless mode (containerd worker)
+
+[RootlessKit](https://github.com/rootless-containers/rootlesskit/) needs to be installed.
+
+Run containerd in rootless mode using rootlesskit following [containerd's document](https://github.com/containerd/containerd/blob/main/docs/rootless.md).
+
+```
+$ containerd-rootless.sh
+```
+
+Then let buildkitd join the same namespace as containerd.
+
+```
+$ containerd-rootless-setuptool.sh nsenter -- buildkitd --oci-worker=false --containerd-worker=true --containerd-worker-snapshotter=native
+```
+
 ## Troubleshooting
 
 ### Error related to `overlayfs`

executor/containerdexecutor/executor.go

@@ -25,6 +25,7 @@ import (
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/network"
+	rootlessspecconv "github.com/moby/buildkit/util/rootless/specconv"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 )
@@ -39,10 +40,11 @@ type containerdExecutor struct {
 	mu               sync.Mutex
 	apparmorProfile  string
 	traceSocket      string
+	rootless         bool
 }
 
 // New creates a new executor backed by connection to containerd API
-func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, traceSocket string) executor.Executor {
+func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, traceSocket string, rootless bool) executor.Executor {
 	// clean up old hosts/resolv.conf file. ignore errors
 	os.RemoveAll(filepath.Join(root, "hosts"))
 	os.RemoveAll(filepath.Join(root, "resolv.conf"))
@@ -56,6 +58,7 @@ func New(client *containerd.Client, root, cgroup string, networkProviders map[pb
 		running:          make(map[string]chan error),
 		apparmorProfile:  apparmorProfile,
 		traceSocket:      traceSocket,
+		rootless:         rootless,
 	}
 }
 
@@ -164,6 +167,11 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
 	}
 	defer cleanup()
 	spec.Process.Terminal = meta.Tty
+	if w.rootless {
+		if err := rootlessspecconv.ToRootless(spec); err != nil {
+			return err
+		}
+	}
 
 	container, err := w.client.NewContainer(ctx, id,
 		containerd.WithSpec(spec),

util/testutil/integration/containerd.go

@@ -10,10 +10,12 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 func InitContainerdWorker() {
@@ -40,6 +42,23 @@ func InitContainerdWorker() {
 		}
 	}
 
+	// the rootless uid is defined in Dockerfile
+	if s := os.Getenv("BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR"); s != "" {
+		var uid, gid int
+		if _, err := fmt.Sscanf(s, "%d:%d", &uid, &gid); err != nil {
+			logrus.Fatalf("unexpected BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR: %q", s)
+		}
+		if rootlessSupported(uid) {
+			Register(&containerd{
+				name:        "containerd-rootless",
+				containerd:  "containerd",
+				uid:         uid,
+				gid:         gid,
+				snapshotter: "native", // TODO: test with fuse-overlayfs as well, or automatically determine snapshotter
+			})
+		}
+	}
+
 	if s := os.Getenv("BUILDKIT_INTEGRATION_SNAPSHOTTER"); s != "" {
 		Register(&containerd{
 			name:        fmt.Sprintf("containerd-snapshotter-%s", s),
@@ -53,6 +72,8 @@ type containerd struct {
 	name        string
 	containerd  string
 	snapshotter string
+	uid         int
+	gid         int
 	extraEnv    []string // e.g. "PATH=/opt/containerd-1.4/bin:/usr/bin:..."
 }
 
@@ -81,10 +102,23 @@ func (c *containerd) New(ctx context.Context, cfg *BackendConfig) (b Backend, cl
 		}
 	}()
 
+	rootless := false
+	if c.uid != 0 {
+		if c.gid == 0 {
+			return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", c.uid, c.gid)
+		}
+		rootless = true
+	}
+
 	tmpdir, err := ioutil.TempDir("", "bktest_containerd")
 	if err != nil {
 		return nil, nil, err
 	}
+	if rootless {
+		if err := os.Chown(tmpdir, c.uid, c.gid); err != nil {
+			return nil, nil, err
+		}
+	}
 
 	deferF.append(func() error { return os.RemoveAll(tmpdir) })
 
@@ -128,7 +162,14 @@ disabled_plugins = ["cri"]
 		return nil, nil, err
 	}
 
-	cmd := exec.Command(c.containerd, "--config", configFile)
+	containerdArgs := []string{c.containerd, "--config", configFile}
+	rootlessKitState := filepath.Join(tmpdir, "rootlesskit-containerd")
+	if rootless {
+		containerdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.uid), "-i", "--", "exec",
+			"rootlesskit", "--copy-up=/run", "--state-dir", rootlessKitState}, containerdArgs...)
+	}
+
+	cmd := exec.Command(containerdArgs[0], containerdArgs[1:]...)
 	cmd.Env = append(os.Environ(), c.extraEnv...)
 
 	ctdStop, err := startCmd(cmd, cfg.Logs)
@@ -152,7 +193,20 @@ disabled_plugins = ["cri"]
 	if runtime.GOOS != "windows" && c.snapshotter != "native" {
 		c.extraEnv = append(c.extraEnv, "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=true")
 	}
-	buildkitdSock, stop, err := runBuildkitd(ctx, cfg, buildkitdArgs, cfg.Logs, 0, 0, c.extraEnv)
+	if rootless {
+		pidStr, err := os.ReadFile(filepath.Join(rootlessKitState, "child_pid"))
+		if err != nil {
+			return nil, nil, err
+		}
+		pid, err := strconv.ParseInt(string(pidStr), 10, 64)
+		if err != nil {
+			return nil, nil, err
+		}
+		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.uid), "-i", "--", "exec",
+			"nsenter", "-U", "--preserve-credentials", "-m", "-t", fmt.Sprintf("%d", pid)},
+			append(buildkitdArgs, "--containerd-worker-snapshotter=native")...)
+	}
+	buildkitdSock, stop, err := runBuildkitd(ctx, cfg, buildkitdArgs, cfg.Logs, c.uid, c.gid, c.extraEnv)
 	if err != nil {
 		printLogs(cfg.Logs, log.Println)
 		return nil, nil, err
@@ -162,7 +216,7 @@ disabled_plugins = ["cri"]
 	return backend{
 		address:           buildkitdSock,
 		containerdAddress: address,
-		rootless:          false,
+		rootless:          rootless,
 		snapshotter:       c.snapshotter,
 	}, cl, nil
 }

worker/containerd/containerd.go

@@ -26,16 +26,16 @@ import (
 )
 
 // NewWorkerOpt creates a WorkerOpt.
-func NewWorkerOpt(root string, address, snapshotterName, ns string, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string, opts ...containerd.ClientOpt) (base.WorkerOpt, error) {
+func NewWorkerOpt(root string, address, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string, opts ...containerd.ClientOpt) (base.WorkerOpt, error) {
 	opts = append(opts, containerd.WithDefaultNamespace(ns))
 	client, err := containerd.New(address, opts...)
 	if err != nil {
 		return base.WorkerOpt{}, errors.Wrapf(err, "failed to connect client to %q . make sure containerd is running", address)
 	}
-	return newContainerd(root, client, snapshotterName, ns, labels, dns, nopt, apparmorProfile, parallelismSem, traceSocket)
+	return newContainerd(root, client, snapshotterName, ns, rootless, labels, dns, nopt, apparmorProfile, parallelismSem, traceSocket)
 }
 
-func newContainerd(root string, client *containerd.Client, snapshotterName, ns string, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string) (base.WorkerOpt, error) {
+func newContainerd(root string, client *containerd.Client, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string) (base.WorkerOpt, error) {
 	if strings.Contains(snapshotterName, "/") {
 		return base.WorkerOpt{}, errors.Errorf("bad snapshotter name: %q", snapshotterName)
 	}
@@ -122,7 +122,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s
 		ID:             id,
 		Labels:         xlabels,
 		MetadataStore:  md,
-		Executor:       containerdexecutor.New(client, root, "", np, dns, apparmorProfile, traceSocket),
+		Executor:       containerdexecutor.New(client, root, "", np, dns, apparmorProfile, traceSocket, rootless),
 		Snapshotter:    snap,
 		ContentStore:   cs,
 		Applier:        winlayers.NewFileSystemApplierWithWindows(cs, df),

worker/containerd/containerd_test.go

@@ -32,7 +32,8 @@ func newWorkerOpt(t *testing.T, addr string) (base.WorkerOpt, func()) {
 	tmpdir, err := ioutil.TempDir("", "workertest")
 	require.NoError(t, err)
 	cleanup := func() { os.RemoveAll(tmpdir) }
-	workerOpt, err := NewWorkerOpt(tmpdir, addr, "overlayfs", "buildkit-test", nil, nil, netproviders.Opt{Mode: "host"}, "", nil, "")
+	rootless := false
+	workerOpt, err := NewWorkerOpt(tmpdir, addr, "overlayfs", "buildkit-test", rootless, nil, nil, netproviders.Opt{Mode: "host"}, "", nil, "")
 	require.NoError(t, err)
 	return workerOpt, cleanup
 }
@@ -44,6 +45,9 @@ func checkRequirement(t *testing.T) {
 }
 
 func testContainerdWorkerExec(t *testing.T, sb integration.Sandbox) {
+	if sb.Rootless() {
+		t.Skip("requires root")
+	}
 	workerOpt, cleanupWorkerOpt := newWorkerOpt(t, sb.ContainerdAddress())
 	defer cleanupWorkerOpt()
 	w, err := base.NewWorker(context.TODO(), workerOpt)
@@ -53,6 +57,9 @@ func testContainerdWorkerExec(t *testing.T, sb integration.Sandbox) {
 }
 
 func testContainerdWorkerExecFailures(t *testing.T, sb integration.Sandbox) {
+	if sb.Rootless() {
+		t.Skip("requires root")
+	}
 	workerOpt, cleanupWorkerOpt := newWorkerOpt(t, sb.ContainerdAddress())
 	defer cleanupWorkerOpt()
 	w, err := base.NewWorker(context.TODO(), workerOpt)