From 604387df60c8a661dfa90af254c6c67c3b68deae Mon Sep 17 00:00:00 2001 From: Laurent Goderre Date: Thu, 24 Oct 2024 11:12:18 -0400 Subject: [PATCH] Return the full list of descriptors in build metadata Signed-off-by: Laurent Goderre --- cmd/buildctl/build.go | 7 +++++-- cmd/buildctl/build_test.go | 12 +++++++---- exporter/containerimage/export.go | 9 +++++++- exporter/containerimage/exptypes/types.go | 1 + exporter/containerimage/writer.go | 25 +++++++++++++++-------- exporter/oci/export.go | 9 +++++++- 6 files changed, 46 insertions(+), 17 deletions(-) diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index a6ca8feb4f15..e203de0f3c3e 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -439,8 +439,11 @@ func writeMetadataFile(filename string, exporterResponse map[string]string) erro } var raw map[string]interface{} if err = json.Unmarshal(dt, &raw); err != nil || len(raw) == 0 { - out[k] = v - continue + var rawList []map[string]interface{} + if err = json.Unmarshal(dt, &rawList); err != nil || len(rawList) == 0 { + out[k] = v + continue + } } out[k] = json.RawMessage(dt) } diff --git a/cmd/buildctl/build_test.go b/cmd/buildctl/build_test.go index a19b14adc5bd..d85ceb688a01 100644 --- a/cmd/buildctl/build_test.go +++ b/cmd/buildctl/build_test.go @@ -140,7 +140,7 @@ func testBuildMetadataFile(t *testing.T, sb integration.Sandbox) { metadataBytes, err := os.ReadFile(metadataFile) require.NoError(t, err) - var metadata map[string]interface{} + var metadata map[string]json.RawMessage err = json.Unmarshal(metadataBytes, &metadata) require.NoError(t, err) @@ -153,13 +153,17 @@ func testBuildMetadataFile(t *testing.T, sb integration.Sandbox) { require.Contains(t, metadata, exptypes.ExporterImageDescriptorKey) var desc *ocispecs.Descriptor - dtdesc, err := json.Marshal(metadata[exptypes.ExporterImageDescriptorKey]) - require.NoError(t, err) - err = json.Unmarshal(dtdesc, &desc) + err = json.Unmarshal(metadata[exptypes.ExporterImageDescriptorKey], &desc) require.NoError(t, err) require.NotEmpty(t, desc.MediaType) require.NotEmpty(t, desc.Digest.String()) + require.Contains(t, metadata, exptypes.ExporterImageDescriptorsKey) + var descList []*ocispecs.Descriptor + require.NoError(t, err) + err = json.Unmarshal(metadata[exptypes.ExporterImageDescriptorsKey], &descList) + require.NoError(t, err) + cdAddress := sb.ContainerdAddress() if cdAddress == "" { t.Log("no containerd worker, skipping digest verification") diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 02bd91b0c0e9..23546ff6b1cd 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -230,10 +230,11 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } }() - desc, err := e.opt.ImageWriter.Commit(ctx, src, sessionID, inlineCache, &opts) + descriptors, err := e.opt.ImageWriter.Commit(ctx, src, sessionID, inlineCache, &opts) if err != nil { return nil, nil, err } + desc := descriptors[0] defer func() { if err == nil { descref = NewDescriptorReference(*desc, done) @@ -359,6 +360,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + dtdesclist, err := json.Marshal(descriptors) + if err != nil { + return nil, nil, err + } + resp[exptypes.ExporterImageDescriptorsKey] = base64.StdEncoding.EncodeToString(dtdesclist) + return resp, nil, nil } diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index 056485b66f12..cdac05c72307 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -13,6 +13,7 @@ const ( ExporterImageConfigKey = "containerimage.config" ExporterImageConfigDigestKey = "containerimage.config.digest" ExporterImageDescriptorKey = "containerimage.descriptor" + ExporterImageDescriptorsKey = "containerimage.descriptors" ExporterImageBaseConfigKey = "containerimage.base.config" ExporterPlatformsKey = "refs.platforms" ) diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 4e3e678fd7e6..a23cefa7e9ac 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -61,7 +61,7 @@ type ImageWriter struct { opt WriterOpt } -func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, inlineCache exptypes.InlineCache, opts *ImageCommitOpts) (*ocispecs.Descriptor, error) { +func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, inlineCache exptypes.InlineCache, opts *ImageCommitOpts) ([]*ocispecs.Descriptor, error) { if _, ok := inp.Metadata[exptypes.ExporterPlatformsKey]; len(inp.Refs) > 0 && !ok { return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping") } @@ -180,7 +180,10 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } mfstDesc.Annotations[exptypes.ExporterConfigDigestKey] = configDesc.Digest.String() - return mfstDesc, nil + return []*ocispecs.Descriptor{ + mfstDesc, + configDesc, + }, nil } if len(inp.Attestations) > 0 { @@ -225,6 +228,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session labels := map[string]string{} + var descriptors []*ocispecs.Descriptor var attestationManifests []ocispecs.Descriptor for i, p := range ps.Platforms { @@ -261,15 +265,16 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session inlineCacheEntry, _ = inlineCacheResult.FindRef(p.ID) } - desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCacheEntry, opts.Epoch, session.NewGroup(sessionID), baseImg) + mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCacheEntry, opts.Epoch, session.NewGroup(sessionID), baseImg) if err != nil { return nil, err } dp := p.Platform - desc.Platform = &dp - idx.Manifests = append(idx.Manifests, *desc) + mfstDesc.Platform = &dp + idx.Manifests = append(idx.Manifests, *mfstDesc) + descriptors = append(descriptors, mfstDesc, configDesc) - labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = desc.Digest.String() + labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = mfstDesc.Digest.String() if attestations, ok := inp.Attestations[p.ID]; ok { attestations, err := attestation.Unbundle(ctx, session.NewGroup(sessionID), attestations) @@ -304,7 +309,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } defaultSubjects = append(defaultSubjects, intoto.Subject{ Name: pl, - Digest: result.ToDigestMap(desc.Digest), + Digest: result.ToDigestMap(mfstDesc.Digest), }) } stmts, err := attestation.MakeInTotoStatements(ctx, session.NewGroup(sessionID), attestations, defaultSubjects) @@ -312,7 +317,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return nil, err } - desc, err := ic.commitAttestationsManifest(ctx, opts, desc.Digest.String(), stmts) + desc, err := ic.commitAttestationsManifest(ctx, opts, mfstDesc.Digest.String(), stmts) if err != nil { return nil, err } @@ -323,6 +328,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session for i, mfst := range attestationManifests { idx.Manifests = append(idx.Manifests, mfst) + descriptors = append(descriptors, &mfst) labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", len(ps.Platforms)+i)] = mfst.Digest.String() } @@ -344,8 +350,9 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return nil, idxDone(errors.Wrapf(err, "error writing manifest list blob %s", idxDigest)) } idxDone(nil) + descriptors = append([]*ocispecs.Descriptor{&idxDesc}, descriptors...) - return &idxDesc, nil + return descriptors, nil } func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefConfig, s session.Group, refs ...cache.ImmutableRef) ([]solver.Remote, error) { diff --git a/exporter/oci/export.go b/exporter/oci/export.go index df8e4134eaa5..5e86a149e636 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -159,10 +159,11 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } }() - desc, err := e.opt.ImageWriter.Commit(ctx, src, sessionID, inlineCache, &opts) + descriptors, err := e.opt.ImageWriter.Commit(ctx, src, sessionID, inlineCache, &opts) if err != nil { return nil, nil, err } + desc := descriptors[0] defer func() { if err == nil { descref = containerimage.NewDescriptorReference(*desc, done) @@ -194,6 +195,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + dtdesclist, err := json.Marshal(descriptors) + if err != nil { + return nil, nil, err + } + resp[exptypes.ExporterImageDescriptorsKey] = base64.StdEncoding.EncodeToString(dtdesclist) + if n, ok := src.Metadata["image.name"]; e.opts.ImageName == "*" && ok { e.opts.ImageName = string(n) }