dockerfile: add subrequest for dumping LLB contents for dockerfile

This subrequest is similar to `buildctl debug dump-llb` but works on a
running buildkit daemon with the dockerfile frontend. The
`--call=dumpllb` option can be used to dump the raw LLB along with the
metadata and anything else in the definition to a JSON format.

This is useful for debugging purposes.

Signed-off-by: Jonathan A. Sternberg <[email protected]>

Author: jsternberg

frontend/dockerfile/builder/build.go

@@ -16,6 +16,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerui"
 	"github.com/moby/buildkit/frontend/gateway/client"
 	gwpb "github.com/moby/buildkit/frontend/gateway/pb"
+	"github.com/moby/buildkit/frontend/subrequests/dumpllb"
 	"github.com/moby/buildkit/frontend/subrequests/lint"
 	"github.com/moby/buildkit/frontend/subrequests/outline"
 	"github.com/moby/buildkit/frontend/subrequests/targets"
@@ -96,6 +97,9 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
 		Lint: func(ctx context.Context) (*lint.LintResults, error) {
 			return dockerfile2llb.DockerfileLint(ctx, src.Data, convertOpt)
 		},
+		DumpLLB: func(ctx context.Context) (*dumpllb.Result, error) {
+			return dockerfile2llb.DockerfileDumpLLB(ctx, src.Data, convertOpt)
+		},
 	}); err != nil {
 		return nil, err
 	} else if ok {

frontend/dockerfile/dockerfile2llb/convert.go

@@ -30,6 +30,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/moby/buildkit/frontend/dockerfile/shell"
 	"github.com/moby/buildkit/frontend/dockerui"
+	"github.com/moby/buildkit/frontend/subrequests/dumpllb"
 	"github.com/moby/buildkit/frontend/subrequests/lint"
 	"github.com/moby/buildkit/frontend/subrequests/outline"
 	"github.com/moby/buildkit/frontend/subrequests/targets"
@@ -123,6 +124,14 @@ func Dockerfile2Outline(ctx context.Context, dt []byte, opt ConvertOpt) (*outlin
 	return &o, nil
 }
 
+func DockerfileDumpLLB(ctx context.Context, dt []byte, opt ConvertOpt) (*dumpllb.Result, error) {
+	ds, err := toDispatchState(ctx, dt, opt)
+	if err != nil {
+		return nil, err
+	}
+	return ds.DumpLLB(ctx)
+}
+
 func DockerfileLint(ctx context.Context, dt []byte, opt ConvertOpt) (*lint.LintResults, error) {
 	results := &lint.LintResults{}
 	sourceIndex := results.AddSource(opt.SourceMap)
@@ -1223,7 +1232,7 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
 	// Run command can potentially access any file. Mark the full filesystem as used.
 	d.paths["/"] = struct{}{}
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if len(c.Files) > 0 {
 		if len(args) != 1 || !c.PrependShell {
 			return errors.Errorf("parsing produced an invalid run command: %v", args)
@@ -1760,7 +1769,7 @@ func dispatchOnbuild(d *dispatchState, c *instructions.OnbuildCommand) error {
 func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Linter) error {
 	validateUsedOnce(c, &d.cmd, lint)
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if c.PrependShell {
 		if len(d.image.Config.Shell) == 0 {
 			msg := linter.RuleJSONArgsRecommended.Format(c.Name())
@@ -1776,7 +1785,7 @@ func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Lint
 func dispatchEntrypoint(d *dispatchState, c *instructions.EntrypointCommand, lint *linter.Linter) error {
 	validateUsedOnce(c, &d.entrypoint, lint)
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if c.PrependShell {
 		if len(d.image.Config.Shell) == 0 {
 			msg := linter.RuleJSONArgsRecommended.Format(c.Name())

frontend/dockerfile/dockerfile2llb/dumpllb.go

@@ -0,0 +1,32 @@
+package dockerfile2llb
+
+import (
+	"context"
+
+	"github.com/moby/buildkit/frontend/subrequests/dumpllb"
+	"github.com/moby/buildkit/solver/pb"
+	digest "github.com/opencontainers/go-digest"
+)
+
+func (ds *dispatchState) DumpLLB(ctx context.Context) (*dumpllb.Result, error) {
+	def, err := ds.state.Marshal(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	res := &dumpllb.Result{
+		Def:      make(map[digest.Digest]*pb.Op, len(def.Def)),
+		Metadata: def.Metadata,
+		Source:   def.Source,
+	}
+	for _, dt := range def.Def {
+		var op pb.Op
+		if err := op.UnmarshalVT(dt); err != nil {
+			return nil, err
+		}
+
+		dgst := digest.FromBytes(dt)
+		res.Def[dgst] = &op
+	}
+	return res, nil
+}

frontend/dockerui/requests.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/frontend/subrequests"
+	"github.com/moby/buildkit/frontend/subrequests/dumpllb"
 	"github.com/moby/buildkit/frontend/subrequests/lint"
 	"github.com/moby/buildkit/frontend/subrequests/outline"
 	"github.com/moby/buildkit/frontend/subrequests/targets"
@@ -21,6 +22,7 @@ type RequestHandler struct {
 	Outline     func(context.Context) (*outline.Outline, error)
 	ListTargets func(context.Context) (*targets.List, error)
 	Lint        func(context.Context) (*lint.LintResults, error)
+	DumpLLB     func(context.Context) (*dumpllb.Result, error)
 	AllowOther  bool
 }
 
@@ -69,6 +71,16 @@ func (bc *Client) HandleSubrequest(ctx context.Context, h RequestHandler) (*clie
 			res, err := warnings.ToResult(nil)
 			return res, true, err
 		}
+	case dumpllb.SubrequestDumpLLBDefinition.Name:
+		if f := h.DumpLLB; f != nil {
+			result, err := f(ctx)
+			if err != nil {
+				return nil, false, err
+			}
+
+			res, err := result.ToResult()
+			return res, true, err
+		}
 	}
 	if h.AllowOther {
 		return nil, false, nil

frontend/subrequests/dumpllb/dumpllb.go

@@ -0,0 +1,67 @@
+package dumpllb
+
+import (
+	"encoding/json"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/moby/buildkit/frontend/subrequests"
+	"github.com/moby/buildkit/solver/pb"
+	"github.com/opencontainers/go-digest"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+const RequestDumpLLB = "frontend.dumpllb"
+
+var SubrequestDumpLLBDefinition = subrequests.Request{
+	Name:        RequestDumpLLB,
+	Version:     "0.1.0",
+	Type:        subrequests.TypeRPC,
+	Description: "Dump LLB output of a Dockerfile",
+	Opts:        []subrequests.Named{},
+	Metadata: []subrequests.Named{
+		{Name: "result.json"},
+	},
+}
+
+type Result struct {
+	Def      map[digest.Digest]*pb.Op         `json:"def"`
+	Metadata map[digest.Digest]llb.OpMetadata `json:"metadata"`
+	Source   *pb.Source                       `json:"source"`
+}
+
+func (result *Result) ToResult() (*client.Result, error) {
+	res := client.NewResult()
+	dt, err := json.MarshalIndent(result, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	res.AddMeta("result.json", dt)
+
+	res.AddMeta("version", []byte(SubrequestDumpLLBDefinition.Version))
+	return res, nil
+}
+
+func (result *Result) MarshalJSON() ([]byte, error) {
+	var jsonResult struct {
+		Def      map[digest.Digest]json.RawMessage `json:"def"`
+		Metadata map[digest.Digest]llb.OpMetadata  `json:"metadata"`
+		Source   json.RawMessage                   `json:"source"`
+	}
+	jsonResult.Def = make(map[digest.Digest]json.RawMessage, len(result.Def))
+	for dgst, op := range result.Def {
+		dt, err := protojson.Marshal(op)
+		if err != nil {
+			return nil, err
+		}
+		jsonResult.Def[dgst] = dt
+	}
+	jsonResult.Metadata = result.Metadata
+
+	src, err := protojson.Marshal(result.Source)
+	if err != nil {
+		return nil, err
+	}
+	jsonResult.Source = src
+	return json.Marshal(&jsonResult)
+}