From b36488e323504ff729de21cc0336e8f94bf06f59 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 9 Sep 2022 07:42:43 +0900 Subject: [PATCH] rootless: support Google Container-Optimized OS Dockerfile has `VOLUME /home/user/.local/share/buildkit` by default too, but the default VOLUME does not work with rootless on Google's Container-Optimized OS as it is mounted with `nosuid,nodev`. So the volume has to be explicitly mounted as an `emptyDir` volume. Tested with GKE Autopilot 1.24.3-gke.200 (kernel 5.10.123+, containerd 1.6.6). Fix issue 879 Thanks to Andrew Grigorev (ei-grad) and Ben Cressey (bcressey). Signed-off-by: Akihiro Suda --- docs/rootless.md | 31 +++++++++++++++++-- examples/kubernetes/README.md | 3 +- .../deployment+service.rootless.yaml | 8 +++++ examples/kubernetes/job.rootless.yaml | 8 +++++ examples/kubernetes/pod.rootless.yaml | 10 ++++++ examples/kubernetes/statefulset.rootless.yaml | 10 ++++++ 6 files changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/rootless.md b/docs/rootless.md index de41b328b259..ee25875e76ee 100644 --- a/docs/rootless.md +++ b/docs/rootless.md @@ -4,6 +4,30 @@ Rootless mode allows running BuildKit daemon as a non-root user. ## Distribution-specific hint Using Ubuntu kernel is recommended. +### Container-Optimized OS from Google +Make sure to have an `emptyDir` volume below: +```yaml +spec: + containers: + - name: buildkitd + volumeMounts: + # Dockerfile has `VOLUME /home/user/.local/share/buildkit` by default too, + # but the default VOLUME does not work with rootless on Google's Container-Optimized OS + # as it is mounted with `nosuid,nodev`. + # https://github.com/moby/buildkit/issues/879#issuecomment-1240347038 + - mountPath: /home/user/.local/share/buildkit + name: buildkitd + volumes: + - name: buildkitd + emptyDir: {} +``` + +See also the [example manifests](#Kubernetes). + +
+Old distributions + +

### Debian GNU/Linux 10 Add `kernel.unprivileged_userns_clone=1` to `/etc/sysctl.conf` (or `/etc/sysctl.d`) and run `sudo sysctl -p`. @@ -16,8 +40,8 @@ This step is not needed for RHEL/CentOS 8 and later. ### Fedora, before kernel 5.13 You may have to disable SELinux, or run BuildKit with `--oci-worker-snapshotter=fuse-overlayfs`. -### Container-Optimized OS from Google -:warning: Currently unsupported. See [#879](https://github.com/moby/buildkit/issues/879). +

+
## Known limitations * Using the `overlayfs` snapshotter requires kernel >= 5.11 or Ubuntu kernel. @@ -77,6 +101,9 @@ $ rootlesskit buildkitd --oci-worker-snapshotter=native ### Error related to `newuidmap` or `/etc/subuid` See https://rootlesscontaine.rs/getting-started/common/subuid/ +### Error `Options:[rbind ro]}]: operation not permitted` +Make sure to mount an `emptyDir` volume on `/home/user/.local/share/buildkit` . + ## Containerized deployment ### Kubernetes diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index 66a36c102a9b..c8973dc5645d 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -7,8 +7,7 @@ This directory contains Kubernetes manifests for `Pod`, `Deployment` (with `Serv * `Job`: good if you don't want to have daemon pods Using Rootless mode (`*.rootless.yaml`) is recommended because Rootless mode image is executed as non-root user (UID 1000) and doesn't need `securityContext.privileged`. - -:warning: Rootless mode may not work on some host kernels. See [`../../docs/rootless.md`](../../docs/rootless.md). +See [`../../docs/rootless.md`](../../docs/rootless.md). See also ["Building Images Efficiently And Securely On Kubernetes With BuildKit" (KubeCon EU 2019)](https://kccnceu19.sched.com/event/MPX5). diff --git a/examples/kubernetes/deployment+service.rootless.yaml b/examples/kubernetes/deployment+service.rootless.yaml index ecc5ef201145..0b554096fde6 100644 --- a/examples/kubernetes/deployment+service.rootless.yaml +++ b/examples/kubernetes/deployment+service.rootless.yaml @@ -63,11 +63,19 @@ spec: - name: certs readOnly: true mountPath: /certs + # Dockerfile has `VOLUME /home/user/.local/share/buildkit` by default too, + # but the default VOLUME does not work with rootless on Google's Container-Optimized OS + # as it is mounted with `nosuid,nodev`. + # https://github.com/moby/buildkit/issues/879#issuecomment-1240347038 + - mountPath: /home/user/.local/share/buildkit + name: buildkitd volumes: # buildkit-daemon-certs must contain ca.pem, cert.pem, and key.pem - name: certs secret: secretName: buildkit-daemon-certs + - name: buildkitd + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/examples/kubernetes/job.rootless.yaml b/examples/kubernetes/job.rootless.yaml index 3910a25906b8..06e608c6ab35 100644 --- a/examples/kubernetes/job.rootless.yaml +++ b/examples/kubernetes/job.rootless.yaml @@ -52,8 +52,16 @@ spec: - name: workspace readOnly: true mountPath: /workspace + # Dockerfile has `VOLUME /home/user/.local/share/buildkit` by default too, + # but the default VOLUME does not work with rootless on Google's Container-Optimized OS + # as it is mounted with `nosuid,nodev`. + # https://github.com/moby/buildkit/issues/879#issuecomment-1240347038 + - mountPath: /home/user/.local/share/buildkit + name: buildkitd # To push the image, you also need to create `~/.docker/config.json` secret # and set $DOCKER_CONFIG to `/path/to/.docker` directory. volumes: - name: workspace emptyDir: {} + - name: buildkitd + emptyDir: {} diff --git a/examples/kubernetes/pod.rootless.yaml b/examples/kubernetes/pod.rootless.yaml index 4e8bff2e15ff..130ea43633fe 100644 --- a/examples/kubernetes/pod.rootless.yaml +++ b/examples/kubernetes/pod.rootless.yaml @@ -34,3 +34,13 @@ spec: # To change UID/GID, you need to rebuild the image runAsUser: 1000 runAsGroup: 1000 + volumeMounts: + # Dockerfile has `VOLUME /home/user/.local/share/buildkit` by default too, + # but the default VOLUME does not work with rootless on Google's Container-Optimized OS + # as it is mounted with `nosuid,nodev`. + # https://github.com/moby/buildkit/issues/879#issuecomment-1240347038 + - mountPath: /home/user/.local/share/buildkit + name: buildkitd + volumes: + - name: buildkitd + emptyDir: {} diff --git a/examples/kubernetes/statefulset.rootless.yaml b/examples/kubernetes/statefulset.rootless.yaml index da2ab0fad57d..0533d2a1004f 100644 --- a/examples/kubernetes/statefulset.rootless.yaml +++ b/examples/kubernetes/statefulset.rootless.yaml @@ -47,3 +47,13 @@ spec: # To change UID/GID, you need to rebuild the image runAsUser: 1000 runAsGroup: 1000 + volumeMounts: + # Dockerfile has `VOLUME /home/user/.local/share/buildkit` by default too, + # but the default VOLUME does not work with rootless on Google's Container-Optimized OS + # as it is mounted with `nosuid,nodev`. + # https://github.com/moby/buildkit/issues/879#issuecomment-1240347038 + - mountPath: /home/user/.local/share/buildkit + name: buildkitd + volumes: + - name: buildkitd + emptyDir: {}