Introduce relocation map generation

- Fixup does not mutate the bundle anymore, unless we want it too using auto-update-bundle,
  then it will update the digest, size and media type sections
- Fixup now generates a relocation map, with original service/invocation images as keys
  and the digested references of the pushed images as values
- Pull now generates the relocation map too. The result of pulling a bundle is a
  bundle.json and a relocation map.
- Bundle is now stored as is in the registry, it is no more re-generated from the
  OCI index. Bundle is stored using the canonical form.
- Split --output flag for fixup and pull commands into 2 flags: --bundle and --relocation-map
- Harden e2e test, checking fixed bundle and generated relocation map

Signed-off-by: Radu M <[email protected]>
Signed-off-by: Silvin Lubecki <[email protected]>

cmd/cnab-to-oci/fixup.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -12,14 +11,17 @@ import (
 	"github.com/docker/cli/cli/config"
 	"github.com/docker/cnab-to-oci/remotes"
 	"github.com/docker/distribution/reference"
+	"github.com/docker/go/canonical/json"
 	"github.com/spf13/cobra"
 )
 
 type fixupOptions struct {
 	input              string
-	output             string
+	bundle             string
+	relocationMap      string
 	targetRef          string
 	insecureRegistries []string
+	autoUpdateBundle   bool
 }
 
 func fixupCmd() *cobra.Command {
@@ -34,9 +36,11 @@ func fixupCmd() *cobra.Command {
 			return runFixup(opts)
 		},
 	}
-	cmd.Flags().StringVarP(&opts.output, "output", "o", "fixed-bundle.json", "specify the output file")
+	cmd.Flags().StringVar(&opts.bundle, "bundle", "fixed-bundle.json", "fixed bundle output file (- to print on standard output)")
+	cmd.Flags().StringVar(&opts.relocationMap, "relocation-map", "relocation-map.json", "relocation map output file (- to print on standard output)")
 	cmd.Flags().StringVarP(&opts.targetRef, "target", "t", "", "reference where the bundle will be pushed")
 	cmd.Flags().StringSliceVar(&opts.insecureRegistries, "insecure-registries", nil, "Use plain HTTP for those registries")
+	cmd.Flags().BoolVar(&opts.autoUpdateBundle, "auto-update-bundle", false, "Updates the bundle image properties with the one resolved on the registry")
 	return cmd
 }
 
@@ -53,18 +57,21 @@ func runFixup(opts fixupOptions) error {
 	if err != nil {
 		return err
 	}
-	if err := remotes.FixupBundle(context.Background(), &b, ref, createResolver(opts.insecureRegistries), remotes.WithEventCallback(displayEvent)); err != nil {
-		return err
+
+	fixupOptions := []remotes.FixupOption{
+		remotes.WithEventCallback(displayEvent),
 	}
-	bundleJSON, err = json.MarshalIndent(b, "", "\t")
+	if opts.autoUpdateBundle {
+		fixupOptions = append(fixupOptions, remotes.WithAutoBundleUpdate())
+	}
+	relocationMap, err := remotes.FixupBundle(context.Background(), &b, ref, createResolver(opts.insecureRegistries), fixupOptions...)
 	if err != nil {
 		return err
 	}
-	if opts.output == "-" {
-		fmt.Fprintln(os.Stdout, string(bundleJSON))
-		return nil
+	if err := writeOutput(opts.bundle, b); err != nil {
+		return err
 	}
-	return ioutil.WriteFile(opts.output, bundleJSON, 0644)
+	return writeOutput(opts.relocationMap, relocationMap)
 }
 
 func displayEvent(ev remotes.FixupEvent) {

cmd/cnab-to-oci/pull.go

@@ -2,18 +2,19 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 
 	"github.com/docker/cnab-to-oci/remotes"
 	"github.com/docker/distribution/reference"
+	"github.com/docker/go/canonical/json"
 	"github.com/spf13/cobra"
 )
 
 type pullOptions struct {
-	output             string
+	bundle             string
+	relocationMap      string
 	targetRef          string
 	insecureRegistries []string
 }
@@ -30,7 +31,8 @@ func pullCmd() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&opts.output, "output", "o", "pulled.json", "output file")
+	cmd.Flags().StringVar(&opts.bundle, "bundle", "pulled.json", "bundle output file (- to print on standard output)")
+	cmd.Flags().StringVar(&opts.relocationMap, "relocation-map", "relocation-map.json", "relocation map output file (- to print on standard output)")
 	cmd.Flags().StringSliceVar(&opts.insecureRegistries, "insecure-registries", nil, "Use plain HTTP for those registries")
 	return cmd
 }
@@ -40,17 +42,25 @@ func runPull(opts pullOptions) error {
 	if err != nil {
 		return err
 	}
-	b, err := remotes.Pull(context.Background(), ref, createResolver(opts.insecureRegistries))
+
+	b, relocationMap, err := remotes.Pull(context.Background(), ref, createResolver(opts.insecureRegistries))
 	if err != nil {
 		return err
 	}
-	bytes, err := json.MarshalIndent(b, "", "\t")
+	if err := writeOutput(opts.bundle, b); err != nil {
+		return err
+	}
+	return writeOutput(opts.relocationMap, relocationMap)
+}
+
+func writeOutput(file string, data interface{}) error {
+	bytes, err := json.MarshalCanonical(data)
 	if err != nil {
 		return err
 	}
-	if opts.output == "-" {
+	if file == "-" {
 		fmt.Fprintln(os.Stdout, string(bytes))
 		return nil
 	}
-	return ioutil.WriteFile(opts.output, bytes, 0644)
+	return ioutil.WriteFile(file, bytes, 0644)
 }

cmd/cnab-to-oci/push.go

@@ -20,6 +20,7 @@ type pushOptions struct {
 	allowFallbacks      bool
 	invocationPlatforms []string
 	componentPlatforms  []string
+	autoUpdateBundle    bool
 }
 
 func pushCmd() *cobra.Command {
@@ -42,6 +43,8 @@ func pushCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&opts.allowFallbacks, "allow-fallbacks", true, "Enable automatic compatibility fallbacks for registries without support for custom media type, or OCI manifests")
 	cmd.Flags().StringSliceVar(&opts.invocationPlatforms, "invocation-platforms", nil, "Platforms to push (for multi-arch invocation images)")
 	cmd.Flags().StringSliceVar(&opts.componentPlatforms, "component-platforms", nil, "Platforms to push (for multi-arch component images)")
+	cmd.Flags().BoolVar(&opts.autoUpdateBundle, "auto-update-bundle", false, "Updates the bundle image properties with the one resolved on the registry")
+
 	return cmd
 }
 
@@ -60,13 +63,19 @@ func runPush(opts pushOptions) error {
 		return err
 	}
 
-	err = remotes.FixupBundle(context.Background(), &b, ref, resolver, remotes.WithEventCallback(displayEvent),
+	fixupOptions := []remotes.FixupOption{
+		remotes.WithEventCallback(displayEvent),
 		remotes.WithInvocationImagePlatforms(opts.invocationPlatforms),
-		remotes.WithComponentImagePlatforms(opts.componentPlatforms))
+		remotes.WithComponentImagePlatforms(opts.componentPlatforms),
+	}
+	if opts.autoUpdateBundle {
+		fixupOptions = append(fixupOptions, remotes.WithAutoBundleUpdate())
+	}
+	relocationMap, err := remotes.FixupBundle(context.Background(), &b, ref, resolver, fixupOptions...)
 	if err != nil {
 		return err
 	}
-	d, err := remotes.Push(context.Background(), &b, ref, resolver, opts.allowFallbacks)
+	d, err := remotes.Push(context.Background(), &b, relocationMap, ref, resolver, opts.allowFallbacks)
 	if err != nil {
 		return err
 	}