Skip to content

Commit 1d4469a

Browse files
tonistiigitonistiigi
authored
Merge pull request #6271 from crazy-max/v0.25_picks_0.25.1
[0.25] cherry-picks for v0.25.1
2 parents 14d1ccb + c35e14d commit 1d4469a

File tree

9 files changed

+217
-26
lines changed

9 files changed

+217
-26
lines changed

‎cache/remotecache/azblob/importer.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
1212
"github.com/containerd/containerd/v2/core/content"
1313
"github.com/containerd/containerd/v2/pkg/labels"
14+
cerrdefs "github.com/containerd/errdefs"
1415
"github.com/moby/buildkit/cache/remotecache"
1516
v1 "github.com/moby/buildkit/cache/remotecache/v1"
1617
"github.com/moby/buildkit/session"
@@ -214,7 +215,7 @@ type ciProvider struct {
214215

215216
func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
216217
if dgst != p.desc.Digest {
217-
return content.Info{}, errors.Errorf("content not found %s", dgst)
218+
return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
218219
}
219220

220221
if p.checked {
@@ -234,7 +235,7 @@ func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info
234235
}
235236

236237
if !exists {
237-
return content.Info{}, errors.Errorf("blob %s not found", dgst)
238+
return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
238239
}
239240

240241
p.checked = true

‎cache/remotecache/gha/gha.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/containerd/containerd/v2/core/content"
1616
"github.com/containerd/containerd/v2/pkg/labels"
17+
cerrdefs "github.com/containerd/errdefs"
1718
"github.com/moby/buildkit/cache/remotecache"
1819
v1 "github.com/moby/buildkit/cache/remotecache/v1"
1920
"github.com/moby/buildkit/session"
@@ -440,7 +441,7 @@ type ciProvider struct {
440441

441442
func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
442443
if dgst != p.desc.Digest {
443-
return content.Info{}, errors.Errorf("content not found %s", dgst)
444+
return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
444445
}
445446

446447
if _, err := p.loadEntry(ctx, p.desc); err != nil {
@@ -465,7 +466,7 @@ func (p *ciProvider) loadEntry(ctx context.Context, desc ocispecs.Descriptor) (*
465466
return nil, err
466467
}
467468
if ce == nil {
468-
return nil, errors.Errorf("blob %s not found", desc.Digest)
469+
return nil, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", desc.Digest)
469470
}
470471
if p.entries == nil {
471472
p.entries = make(map[digest.Digest]*actionscache.Entry)

‎cache/remotecache/v1/chains.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"github.com/cespare/xxhash/v2"
1212
"github.com/containerd/containerd/v2/core/content"
13+
cerrdefs "github.com/containerd/errdefs"
1314
"github.com/moby/buildkit/session"
1415
"github.com/moby/buildkit/solver"
1516
digest "github.com/opencontainers/go-digest"
@@ -246,7 +247,7 @@ func (p DescriptorProviderPair) Info(ctx context.Context, dgst digest.Digest) (c
246247
return p.InfoProvider.Info(ctx, dgst)
247248
}
248249
if dgst != p.Descriptor.Digest {
249-
return content.Info{}, errors.Errorf("content not found %s", dgst)
250+
return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
250251
}
251252
return content.Info{
252253
Digest: p.Descriptor.Digest,

‎docs/buildkitd.toml.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ trace = true
2121
root = "/var/lib/buildkit"
2222
# insecure-entitlements allows insecure entitlements, disabled by default.
2323
insecure-entitlements = [ "network.host", "security.insecure", "device" ]
24+
# provenanceEnvDir is the directory where extra config is loaded that is added
25+
# to the provenance of builds:
26+
# slsa v0.2: invocation.environment.*
27+
# slsa v1: buildDefinition.internalParameters.*
28+
provenanceEnvDir = "/etc/buildkit/provenance.d"
2429

2530
[log]
2631
# log formatter: json or text

‎hack/images‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ done
108108

109109
nocacheFilterFlag=""
110110
if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then
111-
nocacheFilterFlag="--no-cache-filter=buildkit-export,gobuild-base,rootless"
111+
nocacheFilterFlag="--no-cache-filter=buildkit-export-alpine,buildkit-export-ubuntu,gobuild-base,rootless"
112112
fi
113113

114114
buildxCmd build --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" --build-arg BUILDKITD_TAGS --build-arg BUILDKIT_DEBUG --build-arg EXPORT_BASE $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \

‎solver/exporter.go‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"errors"
66
"slices"
77

8+
cerrdefs "github.com/containerd/errdefs"
89
digest "github.com/opencontainers/go-digest"
910
)
1011

@@ -189,24 +190,28 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
189190
if (remote == nil || opt.CompressionOpt != nil) && opt.Mode != CacheExportModeRemoteOnly {
190191
res, err := cm.results.Load(ctx, res)
191192
if err != nil {
192-
return nil, err
193-
}
194-
remotes, err := opt.ResolveRemotes(ctx, res)
195-
if err != nil {
196-
return nil, err
197-
}
198-
res.Release(context.TODO())
199-
if remote == nil && len(remotes) > 0 {
200-
remote, remotes = remotes[0], remotes[1:] // pop the first element
201-
}
202-
if opt.CompressionOpt != nil {
203-
for _, r := range remotes { // record all remaining remotes as well
204-
results = append(results, CacheExportResult{
205-
CreatedAt: v.CreatedAt,
206-
Result: r,
207-
EdgeVertex: k.vtx,
208-
EdgeIndex: k.output,
209-
})
193+
if !errors.Is(err, cerrdefs.ErrNotFound) {
194+
return nil, err
195+
}
196+
remote = nil
197+
} else {
198+
remotes, err := opt.ResolveRemotes(ctx, res)
199+
if err != nil {
200+
return nil, err
201+
}
202+
res.Release(context.TODO())
203+
if remote == nil && len(remotes) > 0 {
204+
remote, remotes = remotes[0], remotes[1:] // pop the first element
205+
}
206+
if opt.CompressionOpt != nil {
207+
for _, r := range remotes { // record all remaining remotes as well
208+
results = append(results, CacheExportResult{
209+
CreatedAt: v.CreatedAt,
210+
Result: r,
211+
EdgeVertex: k.vtx,
212+
EdgeIndex: k.output,
213+
})
214+
}
210215
}
211216
}
212217
}
@@ -232,7 +237,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
232237
for _, dep := range deps {
233238
rec, err := dep.CacheKey.Exporter.ExportTo(ctx, t, opt)
234239
if err != nil {
235-
return nil, err
240+
continue
236241
}
237242
for _, r := range rec {
238243
srcs[i] = append(srcs[i], CacheLink{Src: r, Selector: string(dep.Selector)})
@@ -244,7 +249,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
244249
for _, de := range e.edge.secondaryExporters {
245250
recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt)
246251
if err != nil {
247-
return nil, nil
252+
continue
248253
}
249254
for _, r := range recs {
250255
srcs[de.index] = append(srcs[de.index], CacheLink{Src: r, Selector: de.cacheKey.Selector.String()})
@@ -261,6 +266,14 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
261266
}
262267
}
263268

269+
// validate deps are present
270+
for _, deps := range srcs {
271+
if len(deps) == 0 {
272+
res[e] = nil
273+
return res[e], nil
274+
}
275+
}
276+
264277
if v != nil && len(deps) == 0 {
265278
cm := v.cacheManager
266279
key := cm.getID(v.key)

‎solver/llbsolver/provenance/types/types.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package types
22

33
import (
4+
"encoding/json"
5+
"maps"
46
"slices"
57

68
slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/common"
@@ -311,3 +313,95 @@ func (p *ProvenancePredicateSLSA02) ConvertToSLSA1() *ProvenancePredicateSLSA1 {
311313
RunDetails: runDetails,
312314
}
313315
}
316+
317+
// MarshalJSON flattens ProvenanceCustomEnv into top level.
318+
func (p ProvenanceInternalParametersSLSA1) MarshalJSON() ([]byte, error) {
319+
type Alias ProvenanceInternalParametersSLSA1
320+
base, err := json.Marshal(Alias(p))
321+
if err != nil {
322+
return nil, err
323+
}
324+
var m map[string]any
325+
if err := json.Unmarshal(base, &m); err != nil {
326+
return nil, err
327+
}
328+
maps.Copy(m, p.ProvenanceCustomEnv)
329+
delete(m, "ProvenanceCustomEnv")
330+
return json.Marshal(m)
331+
}
332+
333+
// UnmarshalJSON fills both struct fields and flattened custom env.
334+
func (p *ProvenanceInternalParametersSLSA1) UnmarshalJSON(data []byte) error {
335+
var m map[string]any
336+
if err := json.Unmarshal(data, &m); err != nil {
337+
return err
338+
}
339+
340+
type Alias ProvenanceInternalParametersSLSA1
341+
var a Alias
342+
if err := json.Unmarshal(data, &a); err != nil {
343+
return err
344+
}
345+
346+
// Unmarshal known struct again to identify its keys
347+
structBytes, err := json.Marshal(a)
348+
if err != nil {
349+
return err
350+
}
351+
var known map[string]any
352+
if err := json.Unmarshal(structBytes, &known); err != nil {
353+
return err
354+
}
355+
356+
for k := range known {
357+
delete(m, k)
358+
}
359+
360+
*p = ProvenanceInternalParametersSLSA1(a)
361+
p.ProvenanceCustomEnv = m
362+
return nil
363+
}
364+
365+
func (p Environment) MarshalJSON() ([]byte, error) {
366+
type Alias Environment
367+
base, err := json.Marshal(Alias(p))
368+
if err != nil {
369+
return nil, err
370+
}
371+
var m map[string]any
372+
if err := json.Unmarshal(base, &m); err != nil {
373+
return nil, err
374+
}
375+
maps.Copy(m, p.ProvenanceCustomEnv)
376+
delete(m, "ProvenanceCustomEnv")
377+
return json.Marshal(m)
378+
}
379+
380+
func (p *Environment) UnmarshalJSON(data []byte) error {
381+
var m map[string]any
382+
if err := json.Unmarshal(data, &m); err != nil {
383+
return err
384+
}
385+
386+
type Alias Environment
387+
var a Alias
388+
if err := json.Unmarshal(data, &a); err != nil {
389+
return err
390+
}
391+
// Unmarshal known struct again to identify its keys
392+
structBytes, err := json.Marshal(a)
393+
if err != nil {
394+
return err
395+
}
396+
var known map[string]any
397+
if err := json.Unmarshal(structBytes, &known); err != nil {
398+
return err
399+
}
400+
401+
for k := range known {
402+
delete(m, k)
403+
}
404+
*p = Environment(a)
405+
p.ProvenanceCustomEnv = m
406+
return nil
407+
}

‎solver/llbsolver/provenance/types/types_test.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package types
2+
3+
import (
4+
"encoding/json"
5+
"testing"
6+
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
func TestMarsalBuildDefinitionSLSA1(t *testing.T) {
11+
inp := `{
12+
"buildType": "btype1",
13+
"externalParameters": {
14+
"configSource": {},
15+
"request": {}
16+
},
17+
"internalParameters": {
18+
"builderPlatform": "linux/amd64",
19+
"foo": "bar",
20+
"abc": 123,
21+
"def": {"one": 1}
22+
}
23+
}`
24+
25+
var def ProvenanceBuildDefinitionSLSA1
26+
err := json.Unmarshal([]byte(inp), &def)
27+
require.NoError(t, err)
28+
29+
require.Equal(t, "btype1", def.BuildType)
30+
require.Equal(t, "linux/amd64", def.InternalParameters.BuilderPlatform)
31+
require.Equal(t, "bar", def.InternalParameters.ProvenanceCustomEnv["foo"])
32+
require.InEpsilon(t, float64(123), def.InternalParameters.ProvenanceCustomEnv["abc"], 0.001)
33+
require.Equal(t, map[string]any{"one": float64(1)}, def.InternalParameters.ProvenanceCustomEnv["def"])
34+
35+
out, err := json.Marshal(def)
36+
require.NoError(t, err)
37+
38+
require.JSONEq(t, inp, string(out))
39+
}
40+
41+
func TestMarshalInvocation(t *testing.T) {
42+
inp := `{
43+
"configSource": {
44+
"uri": "git+https://github.com/example/repo.git"
45+
},
46+
"parameters": {
47+
"frontend": "dockerfile.v0"
48+
},
49+
"environment": {
50+
"platform": "linux/amd64",
51+
"buildkit": "v0.10.3",
52+
"custom": {
53+
"foo": "bar"
54+
},
55+
"bar": [1,2,3]
56+
}
57+
}`
58+
59+
var inv ProvenanceInvocationSLSA02
60+
err := json.Unmarshal([]byte(inp), &inv)
61+
require.NoError(t, err)
62+
63+
require.Equal(t, "git+https://github.com/example/repo.git", inv.ConfigSource.URI)
64+
require.Equal(t, "dockerfile.v0", inv.Parameters.Frontend)
65+
require.Equal(t, "linux/amd64", inv.Environment.Platform)
66+
require.Equal(t, "v0.10.3", inv.Environment.ProvenanceCustomEnv["buildkit"])
67+
require.Equal(t, "bar", inv.Environment.ProvenanceCustomEnv["custom"].(map[string]any)["foo"])
68+
require.Equal(t, []any{float64(1), float64(2), float64(3)}, inv.Environment.ProvenanceCustomEnv["bar"])
69+
out, err := json.Marshal(inv)
70+
require.NoError(t, err)
71+
72+
require.JSONEq(t, inp, string(out))
73+
}

‎solver/llbsolver/solver.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,7 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter
847847
eg, ctx := errgroup.WithContext(ctx)
848848
resps := make([]map[string]string, len(exporters))
849849
descs := make([]exporter.DescriptorReference, len(exporters))
850+
var inlineCacheMu sync.Mutex
850851
for i, exp := range exporters {
851852
i, exp := i, exp
852853
eg.Go(func() error {
@@ -865,6 +866,8 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter
865866
}
866867
}
867868
inlineCache := exptypes.InlineCache(func(ctx context.Context) (*result.Result[*exptypes.InlineCacheEntry], error) {
869+
inlineCacheMu.Lock() // ensure only one inline cache exporter runs at a time
870+
defer inlineCacheMu.Unlock()
868871
return runInlineCacheExporter(ctx, exp, inlineCacheExporter, job, cached)
869872
})
870873

0 commit comments

Comments
 (0)