Skip to content

Commit 344fdb7

Browse files
tonistiigitonistiigi
committed
imagetools: add platforms filter to imagetools create
Allows specifying platforms that should be included in the new image, making it possible to reduce platforms of existing multi-arch image. Previously the individual image manifests needed to be used as sources, but that dropped their related attestation manifests. Signed-off-by: Tonis Tiigi <[email protected]>
1 parent 4ad48f7 commit 344fdb7

File tree

6 files changed

+175
-22
lines changed

6 files changed

+175
-22
lines changed

‎build/build.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -723,7 +723,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
723723
return err
724724
}
725725

726-
dt, desc, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
726+
dt, desc, _, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
727727
if err != nil {
728728
return err
729729
}

‎commands/imagetools/create.go‎

Lines changed: 146 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,16 @@ import (
77
"os"
88
"strings"
99

10+
"github.com/containerd/containerd/v2/core/images"
11+
"github.com/containerd/platforms"
1012
"github.com/distribution/reference"
1113
"github.com/docker/buildx/builder"
1214
"github.com/docker/buildx/util/buildflags"
1315
"github.com/docker/buildx/util/cobrautil/completion"
1416
"github.com/docker/buildx/util/imagetools"
1517
"github.com/docker/buildx/util/progress"
1618
"github.com/docker/cli/cli/command"
19+
"github.com/moby/buildkit/util/attestation"
1720
"github.com/moby/buildkit/util/progress/progressui"
1821
"github.com/opencontainers/go-digest"
1922
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -31,6 +34,7 @@ type createOptions struct {
3134
actionAppend bool
3235
progress string
3336
preferIndex bool
37+
platforms []string
3438
}
3539

3640
func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, args []string) error {
@@ -67,6 +71,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
6771
return err
6872
}
6973

74+
platforms, err := parsePlatforms(in.platforms)
75+
if err != nil {
76+
return err
77+
}
78+
7079
repos := map[string]struct{}{}
7180

7281
for _, t := range tags {
@@ -160,7 +169,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
160169
return errors.Wrapf(err, "failed to parse annotations")
161170
}
162171

163-
dt, desc, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
172+
dt, desc, srcMap, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
173+
if err != nil {
174+
return err
175+
}
176+
177+
dt, desc, manifests, err := filterPlatforms(dt, desc, srcMap, platforms)
164178
if err != nil {
165179
return err
166180
}
@@ -170,6 +184,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
170184
return nil
171185
}
172186

187+
// manifests can be nil only if pushing one single-platform desc directly
188+
if manifests == nil {
189+
manifests = []descWithSource{{Descriptor: desc, Source: srcs[0]}}
190+
}
191+
173192
// new resolver cause need new auth
174193
r = imagetools.New(imageopt)
175194

@@ -187,17 +206,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
187206
eg.Go(func() error {
188207
return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
189208
eg2, _ := errgroup.WithContext(ctx)
190-
for _, s := range srcs {
191-
if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
192-
continue
193-
}
194-
s := s
209+
for _, desc := range manifests {
195210
eg2.Go(func() error {
196-
sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String()))
197-
return r.Copy(ctx, s, t)
211+
sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", desc.Digest.String(), desc.Source.Ref.String(), t.String()))
212+
return r.Copy(ctx, desc.Source, t)
198213
})
199214
}
200-
201215
if err := eg2.Wait(); err != nil {
202216
return err
203217
}
@@ -216,6 +230,108 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
216230
return err
217231
}
218232

233+
type descWithSource struct {
234+
ocispecs.Descriptor
235+
Source *imagetools.Source
236+
}
237+
238+
func filterPlatforms(dt []byte, desc ocispecs.Descriptor, srcMap map[digest.Digest]*imagetools.Source, plats []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []descWithSource, error) {
239+
if len(plats) == 0 {
240+
return dt, desc, nil, nil
241+
}
242+
243+
matcher := platforms.Any(plats...)
244+
245+
if !images.IsIndexType(desc.MediaType) {
246+
var mfst ocispecs.Manifest
247+
if err := json.Unmarshal(dt, &mfst); err != nil {
248+
return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse manifest")
249+
}
250+
if desc.Platform == nil {
251+
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("cannot filter platforms from a manifest without platform information")
252+
}
253+
if !matcher.Match(*desc.Platform) {
254+
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("input platform %s does not match any of the provided platforms", platforms.Format(*desc.Platform))
255+
}
256+
return dt, desc, nil, nil
257+
}
258+
259+
var idx ocispecs.Index
260+
if err := json.Unmarshal(dt, &idx); err != nil {
261+
return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse index")
262+
}
263+
264+
var manifestMap = map[digest.Digest]ocispecs.Descriptor{}
265+
for _, m := range idx.Manifests {
266+
manifestMap[m.Digest] = m
267+
}
268+
var references = map[digest.Digest]struct{}{}
269+
for _, m := range idx.Manifests {
270+
if refType, ok := m.Annotations[attestation.DockerAnnotationReferenceType]; ok && refType == attestation.DockerAnnotationReferenceTypeDefault {
271+
dgstStr, ok := m.Annotations[attestation.DockerAnnotationReferenceDigest]
272+
if !ok {
273+
continue
274+
}
275+
dgst, err := digest.Parse(dgstStr)
276+
if err != nil {
277+
continue
278+
}
279+
subject, ok := manifestMap[dgst]
280+
if !ok {
281+
continue
282+
}
283+
if subject.Platform == nil || matcher.Match(*subject.Platform) {
284+
references[m.Digest] = struct{}{}
285+
}
286+
}
287+
}
288+
289+
var mfsts []ocispecs.Descriptor
290+
var mfstsWithSource []descWithSource
291+
292+
for _, m := range idx.Manifests {
293+
_, isRef := references[m.Digest]
294+
if isRef || m.Platform == nil || matcher.Match(*m.Platform) {
295+
src, ok := srcMap[m.Digest]
296+
if !ok {
297+
defaultSource, ok := srcMap[desc.Digest]
298+
if !ok {
299+
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest)
300+
}
301+
src = defaultSource
302+
}
303+
mfsts = append(mfsts, m)
304+
mfstsWithSource = append(mfstsWithSource, descWithSource{
305+
Descriptor: m,
306+
Source: src,
307+
})
308+
}
309+
}
310+
if len(mfsts) == len(idx.Manifests) {
311+
// all platforms matched, no need to rewrite index
312+
return dt, desc, mfstsWithSource, nil
313+
}
314+
315+
if len(mfsts) == 0 {
316+
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("none of the manifests match the provided platforms")
317+
}
318+
319+
idx.Manifests = mfsts
320+
idxBytes, err := json.MarshalIndent(&idx, "", " ")
321+
if err != nil {
322+
return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index")
323+
}
324+
325+
desc = ocispecs.Descriptor{
326+
MediaType: desc.MediaType,
327+
Size: int64(len(idxBytes)),
328+
Digest: digest.FromBytes(idxBytes),
329+
Annotations: desc.Annotations,
330+
}
331+
332+
return idxBytes, desc, mfstsWithSource, nil
333+
}
334+
219335
func parseSources(in []string) ([]*imagetools.Source, error) {
220336
out := make([]*imagetools.Source, len(in))
221337
for i, in := range in {
@@ -228,6 +344,26 @@ func parseSources(in []string) ([]*imagetools.Source, error) {
228344
return out, nil
229345
}
230346

347+
func parsePlatforms(in []string) ([]ocispecs.Platform, error) {
348+
out := make([]ocispecs.Platform, 0, len(in))
349+
for _, p := range in {
350+
if arr := strings.Split(p, ","); len(arr) > 1 {
351+
v, err := parsePlatforms(arr)
352+
if err != nil {
353+
return nil, err
354+
}
355+
out = append(out, v...)
356+
continue
357+
}
358+
plat, err := platforms.Parse(p)
359+
if err != nil {
360+
return nil, errors.Wrapf(err, "invalid platform %q", p)
361+
}
362+
out = append(out, plat)
363+
}
364+
return out, nil
365+
}
366+
231367
func parseRefs(in []string) ([]reference.Named, error) {
232368
refs := make([]reference.Named, len(in))
233369
for i, in := range in {
@@ -291,6 +427,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
291427
flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty", "rawjson"). Use plain to show container output`)
292428
flags.StringArrayVarP(&options.annotations, "annotation", "", []string{}, "Add annotation to the image")
293429
flags.BoolVar(&options.preferIndex, "prefer-index", true, "When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy")
430+
flags.StringArrayVarP(&options.platforms, "platform", "p", []string{}, "Filter specified platforms of target image")
294431

295432
return cmd
296433
}

‎docs/reference/buildx_imagetools_create.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Create a new image based on source images
1717
| `-D`, `--debug` | `bool` | | Enable debug logging |
1818
| [`--dry-run`](#dry-run) | `bool` | | Show final image instead of pushing |
1919
| [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file |
20+
| `-p`, `--platform` | `stringArray` | | Filter specified platforms of target image |
2021
| `--prefer-index` | `bool` | `true` | When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy |
2122
| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`, `rawjson`). Use plain to show container output |
2223
| [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image |

‎util/imagetools/create.go‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ type Source struct {
2828
Ref reference.Named
2929
}
3030

31-
func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, error) {
31+
func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, map[digest.Digest]*Source, error) {
3232
eg, ctx := errgroup.WithContext(ctx)
3333

3434
dts := make([][]byte, len(srcs))
@@ -73,7 +73,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
7373
}
7474

7575
if err := eg.Wait(); err != nil {
76-
return nil, ocispecs.Descriptor{}, err
76+
return nil, ocispecs.Descriptor{}, nil, err
7777
}
7878

7979
// on single source, return original bytes
@@ -83,18 +83,22 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
8383
// of preferIndex since if set to true then the source is already in the preferred format, and if false
8484
// it doesn't matter since we're not going to split it into separate manifests
8585
case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
86-
return dts[0], srcs[0].Desc, nil
86+
srcMap := map[digest.Digest]*Source{
87+
srcs[0].Desc.Digest: srcs[0],
88+
}
89+
return dts[0], srcs[0].Desc, srcMap, nil
8790
default:
8891
if !preferIndex {
89-
return dts[0], srcs[0].Desc, nil
92+
return dts[0], srcs[0].Desc, nil, nil
9093
}
9194
}
9295
}
9396

9497
indexes := map[digest.Digest]int{}
98+
sources := map[digest.Digest]*Source{}
9599
descs := make([]ocispecs.Descriptor, 0, len(srcs))
96100

97-
addDesc := func(d ocispecs.Descriptor) {
101+
addDesc := func(d ocispecs.Descriptor, src *Source) {
98102
idx, ok := indexes[d.Digest]
99103
if ok {
100104
old := descs[idx]
@@ -113,20 +117,21 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
113117
indexes[d.Digest] = len(descs)
114118
descs = append(descs, d)
115119
}
120+
sources[d.Digest] = src
116121
}
117122

118123
for i, src := range srcs {
119124
switch src.Desc.MediaType {
120125
case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
121126
var mfst ocispecs.Index
122127
if err := json.Unmarshal(dts[i], &mfst); err != nil {
123-
return nil, ocispecs.Descriptor{}, errors.WithStack(err)
128+
return nil, ocispecs.Descriptor{}, nil, errors.WithStack(err)
124129
}
125130
for _, d := range mfst.Manifests {
126-
addDesc(d)
131+
addDesc(d, src)
127132
}
128133
default:
129-
addDesc(src.Desc)
134+
addDesc(src.Desc, src)
130135
}
131136
}
132137

@@ -163,9 +168,9 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
163168
}
164169
}
165170
case exptypes.AnnotationManifest, "":
166-
return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are not supported yet", k.Type)
171+
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are not supported yet", k.Type)
167172
case exptypes.AnnotationIndexDescriptor:
168-
return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are invalid while creating an image", k.Type)
173+
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are invalid while creating an image", k.Type)
169174
}
170175
}
171176
}
@@ -179,14 +184,14 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
179184
Annotations: indexAnnotation,
180185
}, "", " ")
181186
if err != nil {
182-
return nil, ocispecs.Descriptor{}, errors.Wrap(err, "failed to marshal index")
187+
return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index")
183188
}
184189

185190
return idxBytes, ocispecs.Descriptor{
186191
MediaType: mt,
187192
Size: int64(len(idxBytes)),
188193
Digest: digest.FromBytes(idxBytes),
189-
}, nil
194+
}, sources, nil
190195
}
191196

192197
func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispecs.Descriptor, dt []byte) error {

‎vendor/github.com/moby/buildkit/util/attestation/types.go‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/modules.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,7 @@ github.com/moby/buildkit/util/apicaps
504504
github.com/moby/buildkit/util/apicaps/pb
505505
github.com/moby/buildkit/util/appcontext
506506
github.com/moby/buildkit/util/appdefaults
507+
github.com/moby/buildkit/util/attestation
507508
github.com/moby/buildkit/util/bklog
508509
github.com/moby/buildkit/util/contentutil
509510
github.com/moby/buildkit/util/disk

0 commit comments

Comments
 (0)