diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f696e20b5b03..b5591bd1bbd7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,6 +69,7 @@ jobs: - ./frontend/dockerfile worker: - containerd + - containerd-rootless - containerd-1.5 - containerd-1.4 - containerd-snapshotter-stargz diff --git a/cmd/buildkitd/config/config.go b/cmd/buildkitd/config/config.go index 311e01880bed..1418132b3259 100644 --- a/cmd/buildkitd/config/config.go +++ b/cmd/buildkitd/config/config.go @@ -99,6 +99,8 @@ type ContainerdConfig struct { ApparmorProfile string `toml:"apparmor-profile"` MaxParallelism int `toml:"max-parallelism"` + + Rootless bool `toml:"rootless"` } type GCPolicy struct { diff --git a/cmd/buildkitd/main_containerd_worker.go b/cmd/buildkitd/main_containerd_worker.go index 190aeb27a673..00079676b1bb 100644 --- a/cmd/buildkitd/main_containerd_worker.go +++ b/cmd/buildkitd/main_containerd_worker.go @@ -11,6 +11,7 @@ import ( "time" ctd "github.com/containerd/containerd" + "github.com/containerd/containerd/pkg/userns" "github.com/moby/buildkit/cmd/buildkitd/config" "github.com/moby/buildkit/util/network/cniprovider" "github.com/moby/buildkit/util/network/netproviders" @@ -99,6 +100,19 @@ func init() { Usage: "set the name of the apparmor profile applied to containers", }, } + n := "containerd-worker-rootless" + u := "enable rootless mode" + if userns.RunningInUserNS() { + flags = append(flags, cli.BoolTFlag{ + Name: n, + Usage: u, + }) + } else { + flags = append(flags, cli.BoolFlag{ + Name: n, + Usage: u, + }) + } if defaultConf.Workers.Containerd.GC == nil || *defaultConf.Workers.Containerd.GC { flags = append(flags, cli.BoolTFlag{ @@ -147,13 +161,14 @@ func applyContainerdFlags(c *cli.Context, cfg *config.Config) error { cfg.Workers.Containerd.Enabled = boolOrAuto } - // GlobalBool works for BoolT as well - rootless := c.GlobalBool("rootless") - if rootless { - logrus.Warn("rootless mode is not supported for containerd workers. disabling containerd worker.") - b := false - cfg.Workers.Containerd.Enabled = &b - return nil + if c.GlobalIsSet("rootless") || c.GlobalBool("rootless") { + cfg.Workers.Containerd.Rootless = c.GlobalBool("rootless") + } + if c.GlobalIsSet("containerd-worker-rootless") { + if !userns.RunningInUserNS() || os.Geteuid() > 0 { + return errors.New("rootless mode requires to be executed as the mapped root in a user namespace; you may use RootlessKit for setting up the namespace") + } + cfg.Workers.Containerd.Rootless = c.GlobalBool("containerd-worker-rootless") } labels, err := attrMap(c.GlobalStringSlice("containerd-worker-labels")) @@ -217,6 +232,13 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ return nil, nil } + if cfg.Rootless { + logrus.Debugf("running in rootless mode") + if common.config.Workers.Containerd.NetworkConfig.Mode == "auto" { + common.config.Workers.Containerd.NetworkConfig.Mode = "host" + } + } + dns := getDNSConfig(common.config.DNS) nc := netproviders.Opt{ @@ -237,7 +259,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ if cfg.Snapshotter != "" { snapshotter = cfg.Snapshotter } - opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, parallelismSem, common.traceSocket, ctd.WithTimeout(60*time.Second)) + opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Rootless, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, parallelismSem, common.traceSocket, ctd.WithTimeout(60*time.Second)) if err != nil { return nil, err } diff --git a/docs/rootless.md b/docs/rootless.md index a05f3640d619..de41b328b259 100644 --- a/docs/rootless.md +++ b/docs/rootless.md @@ -24,10 +24,8 @@ You may have to disable SELinux, or run BuildKit with `--oci-worker-snapshotter= On kernel >= 4.18, the `fuse-overlayfs` snapshotter is used instead of `overlayfs`. On kernel < 4.18, the `native` snapshotter is used. * Network mode is always set to `network.host`. -* No support for `containerd` worker. - ("worker" here is a BuildKit term, not a Kubernetes term. Running rootless BuildKit in containerd is fully supported.) -## Running BuildKit in Rootless mode +## Running BuildKit in Rootless mode (OCI worker) [RootlessKit](https://github.com/rootless-containers/rootlesskit/) needs to be installed. @@ -44,6 +42,22 @@ To isolate BuildKit daemon's network namespace from the host (recommended): $ rootlesskit --net=slirp4netns --copy-up=/etc --disable-host-loopback buildkitd ``` +## Running BuildKit in Rootless mode (containerd worker) + +[RootlessKit](https://github.com/rootless-containers/rootlesskit/) needs to be installed. + +Run containerd in rootless mode using rootlesskit following [containerd's document](https://github.com/containerd/containerd/blob/main/docs/rootless.md). + +``` +$ containerd-rootless.sh +``` + +Then let buildkitd join the same namespace as containerd. + +``` +$ containerd-rootless-setuptool.sh nsenter -- buildkitd --oci-worker=false --containerd-worker=true --containerd-worker-snapshotter=native +``` + ## Troubleshooting ### Error related to `overlayfs` diff --git a/executor/containerdexecutor/executor.go b/executor/containerdexecutor/executor.go index 18261b60e86f..7b45df788901 100644 --- a/executor/containerdexecutor/executor.go +++ b/executor/containerdexecutor/executor.go @@ -25,6 +25,7 @@ import ( "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/network" + rootlessspecconv "github.com/moby/buildkit/util/rootless/specconv" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -39,10 +40,11 @@ type containerdExecutor struct { mu sync.Mutex apparmorProfile string traceSocket string + rootless bool } // New creates a new executor backed by connection to containerd API -func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, traceSocket string) executor.Executor { +func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, traceSocket string, rootless bool) executor.Executor { // clean up old hosts/resolv.conf file. ignore errors os.RemoveAll(filepath.Join(root, "hosts")) os.RemoveAll(filepath.Join(root, "resolv.conf")) @@ -56,6 +58,7 @@ func New(client *containerd.Client, root, cgroup string, networkProviders map[pb running: make(map[string]chan error), apparmorProfile: apparmorProfile, traceSocket: traceSocket, + rootless: rootless, } } @@ -164,6 +167,11 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M } defer cleanup() spec.Process.Terminal = meta.Tty + if w.rootless { + if err := rootlessspecconv.ToRootless(spec); err != nil { + return err + } + } container, err := w.client.NewContainer(ctx, id, containerd.WithSpec(spec), diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go index 50c03c2ca2f6..3fb3aa5e33cb 100644 --- a/util/testutil/integration/containerd.go +++ b/util/testutil/integration/containerd.go @@ -10,10 +10,12 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func InitContainerdWorker() { @@ -40,6 +42,23 @@ func InitContainerdWorker() { } } + // the rootless uid is defined in Dockerfile + if s := os.Getenv("BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR"); s != "" { + var uid, gid int + if _, err := fmt.Sscanf(s, "%d:%d", &uid, &gid); err != nil { + logrus.Fatalf("unexpected BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR: %q", s) + } + if rootlessSupported(uid) { + Register(&containerd{ + name: "containerd-rootless", + containerd: "containerd", + uid: uid, + gid: gid, + snapshotter: "native", // TODO: test with fuse-overlayfs as well, or automatically determine snapshotter + }) + } + } + if s := os.Getenv("BUILDKIT_INTEGRATION_SNAPSHOTTER"); s != "" { Register(&containerd{ name: fmt.Sprintf("containerd-snapshotter-%s", s), @@ -53,6 +72,8 @@ type containerd struct { name string containerd string snapshotter string + uid int + gid int extraEnv []string // e.g. "PATH=/opt/containerd-1.4/bin:/usr/bin:..." } @@ -81,10 +102,23 @@ func (c *containerd) New(ctx context.Context, cfg *BackendConfig) (b Backend, cl } }() + rootless := false + if c.uid != 0 { + if c.gid == 0 { + return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", c.uid, c.gid) + } + rootless = true + } + tmpdir, err := ioutil.TempDir("", "bktest_containerd") if err != nil { return nil, nil, err } + if rootless { + if err := os.Chown(tmpdir, c.uid, c.gid); err != nil { + return nil, nil, err + } + } deferF.append(func() error { return os.RemoveAll(tmpdir) }) @@ -128,7 +162,14 @@ disabled_plugins = ["cri"] return nil, nil, err } - cmd := exec.Command(c.containerd, "--config", configFile) + containerdArgs := []string{c.containerd, "--config", configFile} + rootlessKitState := filepath.Join(tmpdir, "rootlesskit-containerd") + if rootless { + containerdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.uid), "-i", "--", "exec", + "rootlesskit", "--copy-up=/run", "--state-dir", rootlessKitState}, containerdArgs...) + } + + cmd := exec.Command(containerdArgs[0], containerdArgs[1:]...) cmd.Env = append(os.Environ(), c.extraEnv...) ctdStop, err := startCmd(cmd, cfg.Logs) @@ -152,7 +193,20 @@ disabled_plugins = ["cri"] if runtime.GOOS != "windows" && c.snapshotter != "native" { c.extraEnv = append(c.extraEnv, "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=true") } - buildkitdSock, stop, err := runBuildkitd(ctx, cfg, buildkitdArgs, cfg.Logs, 0, 0, c.extraEnv) + if rootless { + pidStr, err := os.ReadFile(filepath.Join(rootlessKitState, "child_pid")) + if err != nil { + return nil, nil, err + } + pid, err := strconv.ParseInt(string(pidStr), 10, 64) + if err != nil { + return nil, nil, err + } + buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.uid), "-i", "--", "exec", + "nsenter", "-U", "--preserve-credentials", "-m", "-t", fmt.Sprintf("%d", pid)}, + append(buildkitdArgs, "--containerd-worker-snapshotter=native")...) + } + buildkitdSock, stop, err := runBuildkitd(ctx, cfg, buildkitdArgs, cfg.Logs, c.uid, c.gid, c.extraEnv) if err != nil { printLogs(cfg.Logs, log.Println) return nil, nil, err @@ -162,7 +216,7 @@ disabled_plugins = ["cri"] return backend{ address: buildkitdSock, containerdAddress: address, - rootless: false, + rootless: rootless, snapshotter: c.snapshotter, }, cl, nil } diff --git a/worker/containerd/containerd.go b/worker/containerd/containerd.go index 58a3e1a79091..ffd4b3a6fb91 100644 --- a/worker/containerd/containerd.go +++ b/worker/containerd/containerd.go @@ -26,16 +26,16 @@ import ( ) // NewWorkerOpt creates a WorkerOpt. -func NewWorkerOpt(root string, address, snapshotterName, ns string, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string, opts ...containerd.ClientOpt) (base.WorkerOpt, error) { +func NewWorkerOpt(root string, address, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string, opts ...containerd.ClientOpt) (base.WorkerOpt, error) { opts = append(opts, containerd.WithDefaultNamespace(ns)) client, err := containerd.New(address, opts...) if err != nil { return base.WorkerOpt{}, errors.Wrapf(err, "failed to connect client to %q . make sure containerd is running", address) } - return newContainerd(root, client, snapshotterName, ns, labels, dns, nopt, apparmorProfile, parallelismSem, traceSocket) + return newContainerd(root, client, snapshotterName, ns, rootless, labels, dns, nopt, apparmorProfile, parallelismSem, traceSocket) } -func newContainerd(root string, client *containerd.Client, snapshotterName, ns string, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string) (base.WorkerOpt, error) { +func newContainerd(root string, client *containerd.Client, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, parallelismSem *semaphore.Weighted, traceSocket string) (base.WorkerOpt, error) { if strings.Contains(snapshotterName, "/") { return base.WorkerOpt{}, errors.Errorf("bad snapshotter name: %q", snapshotterName) } @@ -122,7 +122,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s ID: id, Labels: xlabels, MetadataStore: md, - Executor: containerdexecutor.New(client, root, "", np, dns, apparmorProfile, traceSocket), + Executor: containerdexecutor.New(client, root, "", np, dns, apparmorProfile, traceSocket, rootless), Snapshotter: snap, ContentStore: cs, Applier: winlayers.NewFileSystemApplierWithWindows(cs, df), diff --git a/worker/containerd/containerd_test.go b/worker/containerd/containerd_test.go index b93f07d744b7..c3c5286b027d 100644 --- a/worker/containerd/containerd_test.go +++ b/worker/containerd/containerd_test.go @@ -32,7 +32,8 @@ func newWorkerOpt(t *testing.T, addr string) (base.WorkerOpt, func()) { tmpdir, err := ioutil.TempDir("", "workertest") require.NoError(t, err) cleanup := func() { os.RemoveAll(tmpdir) } - workerOpt, err := NewWorkerOpt(tmpdir, addr, "overlayfs", "buildkit-test", nil, nil, netproviders.Opt{Mode: "host"}, "", nil, "") + rootless := false + workerOpt, err := NewWorkerOpt(tmpdir, addr, "overlayfs", "buildkit-test", rootless, nil, nil, netproviders.Opt{Mode: "host"}, "", nil, "") require.NoError(t, err) return workerOpt, cleanup } @@ -44,6 +45,9 @@ func checkRequirement(t *testing.T) { } func testContainerdWorkerExec(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.Skip("requires root") + } workerOpt, cleanupWorkerOpt := newWorkerOpt(t, sb.ContainerdAddress()) defer cleanupWorkerOpt() w, err := base.NewWorker(context.TODO(), workerOpt) @@ -53,6 +57,9 @@ func testContainerdWorkerExec(t *testing.T, sb integration.Sandbox) { } func testContainerdWorkerExecFailures(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.Skip("requires root") + } workerOpt, cleanupWorkerOpt := newWorkerOpt(t, sb.ContainerdAddress()) defer cleanupWorkerOpt() w, err := base.NewWorker(context.TODO(), workerOpt)