diff --git a/cmd/podman/manifest/push.go b/cmd/podman/manifest/push.go index bf25951b44..967bfbbd83 100644 --- a/cmd/podman/manifest/push.go +++ b/cmd/podman/manifest/push.go @@ -56,6 +56,10 @@ func init() { flags.StringVar(&manifestPushOpts.Authfile, authfileFlagName, auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override") _ = pushCmd.RegisterFlagCompletionFunc(authfileFlagName, completion.AutocompleteDefault) + addCompressionFlagName := "add-compression" + flags.StringSliceVar(&manifestPushOpts.AddCompression, addCompressionFlagName, nil, "add instances with selected compression while pushing") + _ = pushCmd.RegisterFlagCompletionFunc(addCompressionFlagName, common.AutocompleteCompressionFormat) + certDirFlagName := "cert-dir" flags.StringVar(&manifestPushOpts.CertDir, certDirFlagName, "", "use certificates at the specified path to access the registry") _ = pushCmd.RegisterFlagCompletionFunc(certDirFlagName, completion.AutocompleteDefault) diff --git a/docs/source/markdown/podman-manifest-push.1.md.in b/docs/source/markdown/podman-manifest-push.1.md.in index fd2f365d4b..fcdcde66e5 100644 --- a/docs/source/markdown/podman-manifest-push.1.md.in +++ b/docs/source/markdown/podman-manifest-push.1.md.in @@ -14,6 +14,17 @@ The list image's ID and the digest of the image's manifest. ## OPTIONS +#### **--add-compression**=*compression* + +Makes sure that requested compression variant for each platform is added to the manifest list keeping original instance +intact in the same manifest list. Supported values are (`gzip`, `zstd` and `zstd:chunked`). Following flag can be used +multiple times. + +Note that `--compression-format` controls the compression format of each instance in the manifest list. `--add-compression` +will add another variant for each instance in the list with the specified compressions. `--compression-format` gzip `--add-compression` +zstd will push a manifest list with each instance being compressed with gzip plus an additional variant of each instance +being compressed with zstd. + #### **--all** Push the images mentioned in the manifest list or image index, in addition to diff --git a/go.mod b/go.mod index e31e1456db..10ac027149 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/containers/buildah v1.31.1-0.20230722114901-5ece066f82c6 github.com/containers/common v0.55.1-0.20230801150045-44bfd82e3ed2 github.com/containers/conmon v2.0.20+incompatible - github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac + github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a github.com/containers/libhvee v0.4.0 github.com/containers/ocicrypt v1.1.7 github.com/containers/psgo v1.8.0 @@ -90,7 +90,7 @@ require ( github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect github.com/coreos/go-oidc/v3 v3.6.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744 // indirect + github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/disiqueira/gotree/v3 v3.0.2 // indirect diff --git a/go.sum b/go.sum index c221a68e0c..05589f9cd5 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,8 @@ github.com/containers/common v0.55.1-0.20230801150045-44bfd82e3ed2 h1:E6OuqpGlvU github.com/containers/common v0.55.1-0.20230801150045-44bfd82e3ed2/go.mod h1:VlEW0hd11FqVMbWYYjuDCU1+IEqElPZO1RznrRHkxyQ= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= -github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac h1:EQQX+EO+F30H9vJS6vfDXx83Z6OL1YNkO5LN36BtPnM= -github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac/go.mod h1:Zg7m6YHPZRl/wbUDZ6vt+yAyXAjAvALVUelmsIPpMcE= +github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a h1:ZK6GNc7wWP9/CTQySx0TM9VN9p+og4Pfd3Y5aAHrwLk= +github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a/go.mod h1:vsetwKSm1kQayKIWlN7SdGNu/KwcVCgnrhh4Z6Yb75s= github.com/containers/libhvee v0.4.0 h1:HGHIIExgP2PjwjHKKoQM3B+3qakNIZcmmkiAO4luAZE= github.com/containers/libhvee v0.4.0/go.mod h1:fyWDxNQccveTdE3Oe+QRuLbwF/iyV0hDxXqRX5Svlic= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= @@ -297,8 +297,8 @@ github.com/crc-org/vfkit v0.1.1/go.mod h1:vjZiHDacUi0iLosvwyLvqJvJXQhByzlLQbMkdI github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744 h1:MqMnhqqfDsYF2bjxndKIqvISTIRBb1KCzrIwVzKJHe0= -github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= +github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd h1:0av0vtcjA8Hqv5gyWj79CLCFVwOOyBNWPjrfUWceMNg= +github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index b94b937b97..0434d5fbc0 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -333,13 +333,14 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { - All bool `schema:"all"` - CompressionFormat string `schema:"compressionFormat"` - CompressionLevel *int `schema:"compressionLevel"` - Format string `schema:"format"` - RemoveSignatures bool `schema:"removeSignatures"` - TLSVerify bool `schema:"tlsVerify"` - Quiet bool `schema:"quiet"` + All bool `schema:"all"` + CompressionFormat string `schema:"compressionFormat"` + CompressionLevel *int `schema:"compressionLevel"` + Format string `schema:"format"` + RemoveSignatures bool `schema:"removeSignatures"` + TLSVerify bool `schema:"tlsVerify"` + Quiet bool `schema:"quiet"` + AddCompression []string `schema:"addCompression"` }{ // Add defaults here once needed. TLSVerify: true, @@ -373,6 +374,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { options := entities.ImagePushOptions{ All: query.All, Authfile: authfile, + AddCompression: query.AddCompression, CompressionFormat: query.CompressionFormat, CompressionLevel: query.CompressionLevel, Format: query.Format, diff --git a/pkg/api/server/register_manifest.go b/pkg/api/server/register_manifest.go index 90edad1d02..282bcd8d85 100644 --- a/pkg/api/server/register_manifest.go +++ b/pkg/api/server/register_manifest.go @@ -60,6 +60,13 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // type: string // required: true // description: the name or ID of the manifest list + // - in: query + // name: addCompression + // required: false + // description: add existing instances with requested compression algorithms to manifest list + // type: array + // items: + // type: string // - in: path // name: destination // type: string diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 9c3ed2b108..e8ac86ce5b 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -144,6 +144,8 @@ type PushOptions struct { CompressionFormat *string // CompressionLevel is the level to use for the compression of the blobs CompressionLevel *int + // Add existing instances with requested compression algorithms to manifest list + AddCompression []string // Manifest type of the pushed image Format *string // Password for authenticating against the registry. diff --git a/pkg/bindings/images/types_push_options.go b/pkg/bindings/images/types_push_options.go index 550b1f4070..7d5d38e1c9 100644 --- a/pkg/bindings/images/types_push_options.go +++ b/pkg/bindings/images/types_push_options.go @@ -93,6 +93,21 @@ func (o *PushOptions) GetCompressionLevel() int { return *o.CompressionLevel } +// WithAddCompression set field AddCompression to given value +func (o *PushOptions) WithAddCompression(value []string) *PushOptions { + o.AddCompression = value + return o +} + +// GetAddCompression returns value of field AddCompression +func (o *PushOptions) GetAddCompression() []string { + if o.AddCompression == nil { + var z []string + return z + } + return o.AddCompression +} + // WithFormat set field Format to given value func (o *PushOptions) WithFormat(value string) *PushOptions { o.Format = &value diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index c58594d983..cd2062c928 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -244,6 +244,9 @@ type ImagePushOptions struct { // integers in the slice represent 0-indexed layer indices, with support for negative // indexing. i.e. 0 is the first layer, -1 is the last (top-most) layer. OciEncryptLayers *[]int + // If necessary, add clones of existing instances with requested compression algorithms to manifest list + // Note: Following option is only valid for `manifest push` + AddCompression []string } // ImagePushReport is the response from pushing an image. diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 224c8488b9..4acf09038a 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -345,6 +345,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify pushOptions.Writer = opts.Writer pushOptions.CompressionLevel = opts.CompressionLevel + pushOptions.AddCompression = opts.AddCompression compressionFormat := opts.CompressionFormat if compressionFormat == "" { diff --git a/pkg/domain/infra/tunnel/manifest.go b/pkg/domain/infra/tunnel/manifest.go index ca708ab0b0..d2bd4e762e 100644 --- a/pkg/domain/infra/tunnel/manifest.go +++ b/pkg/domain/infra/tunnel/manifest.go @@ -135,7 +135,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin } options := new(images.PushOptions) - options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer) + options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression) if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 14a5abe09c..cba3029691 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -13,8 +13,28 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) +// Internal function to verify instance compression +func verifyInstanceCompression(descriptor []imgspecv1.Descriptor, compression string, arch string) bool { + for _, instance := range descriptor { + if instance.Platform.Architecture != arch { + continue + } + if compression == "zstd" { + // if compression is zstd annotations must contain + val, ok := instance.Annotations["io.github.containers.compression.zstd"] + if ok && val == "true" { + return true + } + } else if len(instance.Annotations) == 0 { + return true + } + } + return false +} + var _ = Describe("Podman manifest", func() { const ( @@ -135,6 +155,63 @@ var _ = Describe("Podman manifest", func() { Expect(session2.OutputToString()).To(Equal(session.OutputToString())) }) + It("push with --add-compression", func() { + if podmanTest.Host.Arch == "ppc64le" { + Skip("No registry image for ppc64le") + } + if isRootless() { + err := podmanTest.RestoreArtifact(REGISTRY_IMAGE) + Expect(err).ToNot(HaveOccurred()) + } + lock := GetPortLock("5000") + defer lock.Unlock() + session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) { + Skip("Cannot start docker registry.") + } + + session = podmanTest.Podman([]string{"build", "--platform", "linux/amd64", "-t", "imageone", "build/basicalpine"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"build", "--platform", "linux/arm64", "-t", "imagetwo", "build/basicalpine"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"manifest", "create", "foobar"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imageone:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imagetwo:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + push := podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--remove-signatures", "foobar", "localhost:5000/list"}) + push.WaitWithDefaultTimeout() + Expect(push).Should(Exit(0)) + output := push.ErrorToString() + // 4 images must be pushed two for gzip and two for zstd + Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list")) + + session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + var index imgspecv1.Index + inspectData := []byte(session.OutputToString()) + err := json.Unmarshal(inspectData, &index) + Expect(err).ToNot(HaveOccurred()) + + Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue()) + }) + It("add --all", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() diff --git a/test/system/012-manifest.bats b/test/system/012-manifest.bats index df90a135f2..6f99454a85 100644 --- a/test/system/012-manifest.bats +++ b/test/system/012-manifest.bats @@ -4,6 +4,39 @@ load helpers load helpers.network load helpers.registry +# Helper function for several of the tests which verifies compression. +# +# Usage: validate_instance_compression INDEX MANIFEST ARCH COMPRESSION +# +# INDEX instance which needs to be verified in +# provided manifest list. +# +# MANIFEST OCI manifest specification in json format +# +# ARCH instance architecture +# +# COMPRESSION compression algorithm name; e.g "zstd". +# +function validate_instance_compression { + case $4 in + + gzip) + run jq -r '.manifests['$1'].annotations' <<< $2 + # annotation is `null` for gzip compression + assert "$output" = "null" ".manifests[$1].annotations (null means gzip)" + ;; + + zstd) + # annotation `'"io.github.containers.compression.zstd": "true"'` must be there for zstd compression + run jq -r '.manifests['$1'].annotations."io.github.containers.compression.zstd"' <<< $2 + assert "$output" = "true" ".manifests[$1].annotations.'io.github.containers.compression.zstd' (io.github.containers.compression.zstd must be set)" + ;; + esac + + run jq -r '.manifests['$1'].platform.architecture' <<< $2 + assert "$output" = $3 ".manifests[$1].platform.architecture" +} + # Regression test for #8931 @test "podman images - bare manifest list" { # Create an empty manifest list and list images. @@ -56,4 +89,46 @@ load helpers.registry is "$output" ".*\"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\"" "Verify --tls-verify=false with REGISTRY_AUTH_FILE works against an insecure registry" } +@test "manifest list --add-compression with zstd" { + if ! type -p skopeo; then + skip "skopeo not available" + fi + skip_if_remote "running a local registry doesn't work with podman-remote" + start_registry + + tmpdir=$PODMAN_TMPDIR/build-test + mkdir -p $tmpdir + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile <