diff --git a/Makefile b/Makefile index 2c3655b..a2c8341 100644 --- a/Makefile +++ b/Makefile @@ -32,3 +32,11 @@ fmt/go: .PHONY: fmt/md fmt/md: go run github.com/Kunde21/markdownfmt/v3/cmd/markdownfmt@v3.1.0 -w ./README.md + +.PHONY: test +test: + go test -v -count=1 ./... + +.PHONY: test-integration +test-integration: + go test -v -count=1 -tags=integration ./integration/ diff --git a/README.md b/README.md index 7ed5bda..fba30bc 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,37 @@ env { > } > } > ``` + +## GPUs + +When passing through GPUs to the inner container, you may end up using associated tooling such as the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/index.html) or the [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/index.html). These will inject required utilities and libraries inside the inner container. You can verify this by directly running (without Envbox) a barebones image like `debian:bookworm` and running `mount` or `nvidia-smi` inside the container. + +Envbox will detect these mounts and pass them inside the inner container it creates, so that GPU-aware tools run inside the inner container can still utilize these libraries. + +## Hacking + +Here's a simple one-liner to run the `codercom/enterprise-minimal:ubuntu` image in Envbox using Docker: + +``` +docker run -it --rm \ + -v /tmp/envbox/docker:/var/lib/coder/docker \ + -v /tmp/envbox/containers:/var/lib/coder/containers \ + -v /tmp/envbox/sysbox:/var/lib/sysbox \ + -v /tmp/envbox/docker:/var/lib/docker \ + -v /usr/src:/usr/src:ro \ + -v /lib/modules:/lib/modules:ro \ + --privileged \ + -e CODER_INNER_IMAGE=codercom/enterprise-minimal:ubuntu \ + -e CODER_INNER_USERNAME=coder \ + envbox:latest /envbox docker +``` + +This will store persistent data under `/tmp/envbox`. + +## Troubleshooting + +### `failed to write to cgroup.procs: write /sys/fs/cgroup/docker//init.scope/cgroup.procs: operation not supported: unknown` + +This issue occurs in Docker if you have `cgroupns-mode` set to `private`. To validate, add `--cgroupns=host` to your `docker run` invocation and re-run. + +To permanently set this as the default in your Docker daemon, add `"default-cgroupns-mode": "host"` to your `/etc/docker/daemon.json` and restart Docker. diff --git a/integration/docker_test.go b/integration/docker_test.go index 9f88b09..6bf0f7e 100644 --- a/integration/docker_test.go +++ b/integration/docker_test.go @@ -240,28 +240,53 @@ func TestDocker(t *testing.T) { require.Equal(t, "1000", strings.TrimSpace(string(out))) // Validate that memory limit is being applied to the inner container. - out, err = integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ + // First check under cgroupv2 path. + if out, err = integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ ContainerID: resource.Container.ID, - Cmd: []string{"cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}, - }) - require.NoError(t, err) - require.Equal(t, expectedMemoryLimit, strings.TrimSpace(string(out))) + Cmd: []string{"cat", "/sys/fs/cgroup/memory.max"}, + }); err == nil { + require.Equal(t, expectedMemoryLimit, strings.TrimSpace(string(out))) + } else { // fall back to cgroupv1 path. + out, err = integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ + ContainerID: resource.Container.ID, + Cmd: []string{"cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}, + }) + require.NoError(t, err) + require.Equal(t, expectedMemoryLimit, strings.TrimSpace(string(out))) + } - periodStr, err := integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ + // Validate the cpu limits are being applied to the inner container. + // First check under cgroupv2 path. + var quota, period int64 + if out, err = integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ ContainerID: resource.Container.ID, - Cmd: []string{"cat", "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us"}, - }) - require.NoError(t, err) - period, err := strconv.ParseInt(strings.TrimSpace(string(periodStr)), 10, 64) - require.NoError(t, err) + Cmd: []string{"cat", "/sys/fs/cgroup/cpu.max"}, + }); err == nil { + // out is in the format "period quota" + // e.g. "100000 100000" + fields := strings.Fields(string(out)) + require.Len(t, fields, 2) + period, err = strconv.ParseInt(fields[0], 10, 64) + require.NoError(t, err) + quota, err = strconv.ParseInt(fields[1], 10, 64) + require.NoError(t, err) + } else { // fall back to cgroupv1 path. + periodStr, err := integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ + ContainerID: resource.Container.ID, + Cmd: []string{"cat", "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us"}, + }) + require.NoError(t, err) + period, err = strconv.ParseInt(strings.TrimSpace(string(periodStr)), 10, 64) + require.NoError(t, err) - quotaStr, err := integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ - ContainerID: resource.Container.ID, - Cmd: []string{"cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"}, - }) - require.NoError(t, err) - quota, err := strconv.ParseInt(strings.TrimSpace(string(quotaStr)), 10, 64) - require.NoError(t, err) + quotaStr, err := integrationtest.ExecInnerContainer(t, pool, integrationtest.ExecConfig{ + ContainerID: resource.Container.ID, + Cmd: []string{"cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"}, + }) + require.NoError(t, err) + quota, err = strconv.ParseInt(strings.TrimSpace(string(quotaStr)), 10, 64) + require.NoError(t, err) + } // Validate that the CPU limit is being applied to the inner container. actualLimit := float64(quota) / float64(period) diff --git a/xunix/gpu.go b/xunix/gpu.go index a494ab5..0708667 100644 --- a/xunix/gpu.go +++ b/xunix/gpu.go @@ -17,9 +17,10 @@ import ( ) var ( - gpuMountRegex = regexp.MustCompile("(?i)(nvidia|vulkan|cuda)") - gpuExtraRegex = regexp.MustCompile("(?i)(libgl|nvidia|vulkan|cuda)") - gpuEnvRegex = regexp.MustCompile("(?i)nvidia") + gpuMountRegex = regexp.MustCompile("(?i)(nvidia|vulkan|cuda)") + gpuExtraRegex = regexp.MustCompile("(?i)(libgl|nvidia|vulkan|cuda)") + gpuEnvRegex = regexp.MustCompile("(?i)nvidia") + sharedObjectRegex = regexp.MustCompile(`\.so(\.[0-9\.]+)?$`) ) func GPUEnvs(ctx context.Context) []string { @@ -103,7 +104,7 @@ func usrLibGPUs(ctx context.Context, log slog.Logger, usrLibDir string) ([]mount return nil } - if filepath.Ext(path) != ".so" || !gpuExtraRegex.MatchString(path) { + if !sharedObjectRegex.MatchString(path) || !gpuExtraRegex.MatchString(path) { return nil } diff --git a/xunix/gpu_test.go b/xunix/gpu_test.go index 4cbf5f0..f8d8d47 100644 --- a/xunix/gpu_test.go +++ b/xunix/gpu_test.go @@ -56,13 +56,13 @@ func TestGPUs(t *testing.T) { expectedUsrLibFiles = []string{ filepath.Join(usrLibMountpoint, "nvidia", "libglxserver_nvidia.so"), filepath.Join(usrLibMountpoint, "libnvidia-ml.so"), + filepath.Join(usrLibMountpoint, "nvidia", "libglxserver_nvidia.so.1"), } // fakeUsrLibFiles are files that should be written to the "mounted" // /usr/lib directory. It includes files that shouldn't be returned. fakeUsrLibFiles = append([]string{ filepath.Join(usrLibMountpoint, "libcurl-gnutls.so"), - filepath.Join(usrLibMountpoint, "nvidia", "libglxserver_nvidia.so.1"), }, expectedUsrLibFiles...) ) @@ -98,7 +98,7 @@ func TestGPUs(t *testing.T) { devices, binds, err := xunix.GPUs(ctx, log, usrLibMountpoint) require.NoError(t, err) require.Len(t, devices, 2, "unexpected 2 nvidia devices") - require.Len(t, binds, 3, "expected 4 nvidia binds") + require.Len(t, binds, 4, "expected 4 nvidia binds") require.Contains(t, binds, mount.MountPoint{ Device: "/dev/sda1", Path: "/usr/local/nvidia",