Merge pull request #418 from tonistiigi/dockerfile-url

dockerfile: support building from a Dockerfile URL

cache/fsutil.go

@@ -2,13 +2,25 @@ package cache
 
 import (
 	"context"
+	"io"
 	"io/ioutil"
+	"os"
 
 	"github.com/containerd/continuity/fs"
 	"github.com/moby/buildkit/snapshot"
 )
 
-func ReadFile(ctx context.Context, ref ImmutableRef, p string) ([]byte, error) {
+type ReadRequest struct {
+	Filename string
+	Range    *FileRange
+}
+
+type FileRange struct {
+	Offset int
+	Length int
+}
+
+func ReadFile(ctx context.Context, ref ImmutableRef, req ReadRequest) ([]byte, error) {
 	mount, err := ref.Mount(ctx, true)
 	if err != nil {
 		return nil, err
@@ -27,15 +39,30 @@ func ReadFile(ctx context.Context, ref ImmutableRef, p string) ([]byte, error) {
 		}
 	}()
 
-	fp, err := fs.RootPath(root, p)
+	fp, err := fs.RootPath(root, req.Filename)
 	if err != nil {
 		return nil, err
 	}
 
-	dt, err := ioutil.ReadFile(fp)
-	if err != nil {
-		return nil, err
+	var dt []byte
+
+	if req.Range == nil {
+		dt, err = ioutil.ReadFile(fp)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		f, err := os.Open(fp)
+		if err != nil {
+			return nil, err
+		}
+		dt, err = ioutil.ReadAll(io.NewSectionReader(f, int64(req.Range.Offset), int64(req.Range.Length)))
+		f.Close()
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	if err := lm.Unmount(); err != nil {
 		return nil, err
 	}

frontend/dockerfile/builder/build.go

@@ -1,6 +1,7 @@
 package builder
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -55,14 +56,44 @@ func Build(ctx context.Context, c client.Client) error {
 		llb.SharedKeyHint(defaultDockerfileName),
 	)
 	var buildContext *llb.State
+	isScratchContext := false
 	if st, ok := detectGitContext(opts[LocalNameContext]); ok {
 		src = *st
 		buildContext = &src
 	} else if httpPrefix.MatchString(opts[LocalNameContext]) {
-		unpack := llb.Image(dockerfile2llb.CopyImage).Run(llb.Shlex("copy --unpack /src/context /out/"), llb.ReadonlyRootFS())
-		unpack.AddMount("/src", llb.HTTP(opts[LocalNameContext], llb.Filename("context")), llb.Readonly)
-		src = unpack.AddMount("/out", llb.Scratch())
-		buildContext = &src
+		httpContext := llb.HTTP(opts[LocalNameContext], llb.Filename("context"))
+		def, err := httpContext.Marshal()
+		if err != nil {
+			return err
+		}
+		ref, err := c.Solve(ctx, client.SolveRequest{
+			Definition: def.ToPB(),
+		}, nil, false)
+		if err != nil {
+			return err
+		}
+
+		dt, err := ref.ReadFile(ctx, client.ReadRequest{
+			Filename: "context",
+			Range: &client.FileRange{
+				Length: 1024,
+			},
+		})
+		if err != nil {
+			return err
+		}
+		if isArchive(dt) {
+			unpack := llb.Image(dockerfile2llb.CopyImage).
+				Run(llb.Shlex("copy --unpack /src/context /out/"), llb.ReadonlyRootFS())
+			unpack.AddMount("/src", httpContext, llb.Readonly)
+			src = unpack.AddMount("/out", llb.Scratch())
+			buildContext = &src
+		} else {
+			filename = "context"
+			src = httpContext
+			buildContext = &src
+			isScratchContext = true
+		}
 	}
 
 	def, err := src.Marshal()
@@ -80,42 +111,48 @@ func Build(ctx context.Context, c client.Client) error {
 			return err
 		}
 
-		dtDockerfile, err = ref.ReadFile(ctx2, filename)
+		dtDockerfile, err = ref.ReadFile(ctx2, client.ReadRequest{
+			Filename: filename,
+		})
 		if err != nil {
 			return err
 		}
 		return nil
 	})
 	var excludes []string
-	eg.Go(func() error {
-		dockerignoreState := buildContext
-		if dockerignoreState == nil {
-			st := llb.Local(LocalNameContext,
-				llb.SessionID(c.SessionID()),
-				llb.IncludePatterns([]string{dockerignoreFilename}),
-				llb.SharedKeyHint(dockerignoreFilename),
-			)
-			dockerignoreState = &st
-		}
-		def, err := dockerignoreState.Marshal()
-		if err != nil {
-			return err
-		}
-		ref, err := c.Solve(ctx2, client.SolveRequest{
-			Definition: def.ToPB(),
-		}, nil, false)
-		if err != nil {
-			return err
-		}
-		dtDockerignore, err := ref.ReadFile(ctx2, dockerignoreFilename)
-		if err == nil {
-			excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dtDockerignore))
+	if !isScratchContext {
+		eg.Go(func() error {
+			dockerignoreState := buildContext
+			if dockerignoreState == nil {
+				st := llb.Local(LocalNameContext,
+					llb.SessionID(c.SessionID()),
+					llb.IncludePatterns([]string{dockerignoreFilename}),
+					llb.SharedKeyHint(dockerignoreFilename),
+				)
+				dockerignoreState = &st
+			}
+			def, err := dockerignoreState.Marshal()
 			if err != nil {
-				return errors.Wrap(err, "failed to parse dockerignore")
+				return err
 			}
-		}
-		return nil
-	})
+			ref, err := c.Solve(ctx2, client.SolveRequest{
+				Definition: def.ToPB(),
+			}, nil, false)
+			if err != nil {
+				return err
+			}
+			dtDockerignore, err := ref.ReadFile(ctx2, client.ReadRequest{
+				Filename: dockerignoreFilename,
+			})
+			if err == nil {
+				excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dtDockerignore))
+				if err != nil {
+					return errors.Wrap(err, "failed to parse dockerignore")
+				}
+			}
+			return nil
+		})
+	}
 
 	if err := eg.Wait(); err != nil {
 		return err
@@ -218,3 +255,22 @@ func detectGitContext(ref string) (*llb.State, bool) {
 	st := llb.Git(parts[0], branch)
 	return &st, true
 }
+
+func isArchive(header []byte) bool {
+	for _, m := range [][]byte{
+		{0x42, 0x5A, 0x68},                   // bzip2
+		{0x1F, 0x8B, 0x08},                   // gzip
+		{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz
+	} {
+		if len(header) < len(m) {
+			continue
+		}
+		if bytes.Equal(m, header[:len(m)]) {
+			return true
+		}
+	}
+
+	r := tar.NewReader(bytes.NewBuffer(header))
+	_, err := r.Next()
+	return err == nil
+}

frontend/dockerfile/dockerfile_test.go

@@ -64,9 +64,62 @@ func TestIntegration(t *testing.T) {
 		testBuiltinArgs,
 		testPullScratch,
 		testSymlinkDestination,
+		testHTTPDockerfile,
 	})
 }
 
+func testHTTPDockerfile(t *testing.T, sb integration.Sandbox) {
+	t.Parallel()
+
+	dockerfile := []byte(`
+FROM busybox
+RUN echo -n "foo-contents" > /foo
+FROM scratch
+COPY --from=0 /foo /foo
+`)
+
+	srcDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(srcDir)
+
+	err = ioutil.WriteFile(filepath.Join(srcDir, "Dockerfile"), dockerfile, 0600)
+	require.NoError(t, err)
+
+	resp := httpserver.Response{
+		Etag:    identity.NewID(),
+		Content: dockerfile,
+	}
+
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/df": resp,
+	})
+	defer server.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	c, err := client.New(sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	_, err = c.Solve(context.TODO(), nil, client.SolveOpt{
+		Frontend: "dockerfile.v0",
+		FrontendAttrs: map[string]string{
+			"context":  server.URL + "/df",
+			"filename": "mydockerfile", // this is bogus, any name should work
+		},
+		Exporter:          client.ExporterLocal,
+		ExporterOutputDir: destDir,
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, "foo-contents", string(dt))
+
+}
+
 func testCmdShell(t *testing.T, sb integration.Sandbox) {
 	t.Parallel()
 

frontend/dockerfile/forward.go

@@ -60,12 +60,21 @@ type ref struct {
 	solver.CachedResult
 }
 
-func (r *ref) ReadFile(ctx context.Context, fp string) ([]byte, error) {
+func (r *ref) ReadFile(ctx context.Context, req client.ReadRequest) ([]byte, error) {
 	ref, err := r.getImmutableRef()
 	if err != nil {
 		return nil, err
 	}
-	return cache.ReadFile(ctx, ref, fp)
+	newReq := cache.ReadRequest{
+		Filename: req.Filename,
+	}
+	if r := req.Range; r != nil {
+		newReq.Range = &cache.FileRange{
+			Offset: r.Offset,
+			Length: r.Length,
+		}
+	}
+	return cache.ReadFile(ctx, ref, newReq)
 }
 
 func (r *ref) getImmutableRef() (cache.ImmutableRef, error) {

frontend/gateway/client/client.go

@@ -16,7 +16,19 @@ type Client interface {
 }
 
 type Reference interface {
-	ReadFile(ctx context.Context, fp string) ([]byte, error)
+	ReadFile(ctx context.Context, req ReadRequest) ([]byte, error)
+	// StatFile(ctx context.Context, req StatRequest) (*StatResponse, error)
+	// ReadDir(ctx context.Context, req ReadDirRequest) ([]*StatResponse, error)
+}
+
+type ReadRequest struct {
+	Filename string
+	Range    *FileRange
+}
+
+type FileRange struct {
+	Offset int
+	Length int
 }
 
 // SolveRequest is same as frontend.SolveRequest but avoiding dependency

frontend/gateway/gateway.go

@@ -315,7 +315,18 @@ func (lbf *llbBridgeForwarder) ReadFile(ctx context.Context, req *pb.ReadFileReq
 	if !ok {
 		return nil, errors.Errorf("invalid ref: %T", ref.Sys())
 	}
-	dt, err := cache.ReadFile(ctx, workerRef.ImmutableRef, req.FilePath)
+
+	newReq := cache.ReadRequest{
+		Filename: req.FilePath,
+	}
+	if r := req.Range; r != nil {
+		newReq.Range = &cache.FileRange{
+			Offset: int(r.Offset),
+			Length: int(r.Length),
+		}
+	}
+
+	dt, err := cache.ReadFile(ctx, workerRef.ImmutableRef, newReq)
 	if err != nil {
 		return nil, err
 	}

frontend/gateway/grpcclient/client.go

@@ -84,8 +84,15 @@ type reference struct {
 	c  *grpcClient
 }
 
-func (r *reference) ReadFile(ctx context.Context, fp string) ([]byte, error) {
-	resp, err := r.c.client.ReadFile(ctx, &pb.ReadFileRequest{FilePath: fp, Ref: r.id})
+func (r *reference) ReadFile(ctx context.Context, req client.ReadRequest) ([]byte, error) {
+	rfr := &pb.ReadFileRequest{FilePath: req.Filename, Ref: r.id}
+	if r := req.Range; r != nil {
+		rfr.Range = &pb.FileRange{
+			Offset: int64(r.Offset),
+			Length: int64(r.Length),
+		}
+	}
+	resp, err := r.c.client.ReadFile(ctx, rfr)
 	if err != nil {
 		return nil, err
 	}