Skip to content

Commit

Permalink
buildah/push/manifest-push: add support for --force-compression
Browse files Browse the repository at this point in the history 
Adds support for `--force-compression` which allows end-users to force
push blobs with the selected compresison in `--compression` option, in
order to make sure that `blobs` of other compression on registry are not
reused.

Is equivalent to: `force-compression` here: https://docs.docker.com/build/exporters/#compression
Closes: containers#4613

Also Implements:
`--compression-format` and `--compression-level` for `manifest push` just like
`podman`'s `manifest push`

Signed-off-by: Aditya R <[email protected]>
  • Loading branch information
@flouthoc
flouthoc committed Aug 11, 2023
1 parent 47c1d6e commit b79361c
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 42 deletions.
31 changes: 23 additions & 8 deletions cmd/buildah/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/containers/common/pkg/auth"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
Expand Down Expand Up @@ -226,6 +227,9 @@ func init() {
flags.StringVar(&manifestPushOpts.certDir, "cert-dir", "", "use certificates at the specified path to access the registry")
flags.StringVar(&manifestPushOpts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
flags.StringVar(&manifestPushOpts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting digest to the file")
flags.BoolVarP(&manifestPushOpts.forceCompressionFormat, "force-compression", "", false, "force re-compressing layers with specified format while pushing")
flags.StringVar(&manifestPushOpts.compressionFormat, "compression-format", "", "compression format to use")
flags.IntVar(&manifestPushOpts.compressionLevel, "compression-level", 0, "compression level to use")
flags.StringVarP(&manifestPushOpts.format, "format", "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)")
flags.StringSliceVar(&manifestPushOpts.addCompression, "add-compression", nil, "add instances with selected compression while pushing")
flags.BoolVarP(&manifestPushOpts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing images")
Expand Down Expand Up @@ -864,6 +868,16 @@ func manifestPushCmd(c *cobra.Command, args []string, opts pushOptions) error {
if err != nil {
return fmt.Errorf("building system context: %w", err)
}
if opts.compressionFormat != "" {
algo, err := compression.AlgorithmByName(opts.compressionFormat)
if err != nil {
return err
}
systemContext.CompressionFormat = &algo
}
if c.Flag("compression-level").Changed {
systemContext.CompressionLevel = &opts.compressionLevel
}

return manifestPush(systemContext, store, listImageSpec, destSpec, opts)
}
Expand Down Expand Up @@ -902,14 +916,15 @@ func manifestPush(systemContext *types.SystemContext, store storage.Store, listI
}

options := manifests.PushOptions{
Store: store,
SystemContext: systemContext,
ImageListSelection: cp.CopySpecificImages,
Instances: nil,
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signBy,
ManifestType: manifestType,
AddCompression: opts.addCompression,
Store: store,
SystemContext: systemContext,
ImageListSelection: cp.CopySpecificImages,
Instances: nil,
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signBy,
ManifestType: manifestType,
AddCompression: opts.addCompression,
ForceCompressionFormat: opts.forceCompressionFormat,
}
if opts.all {
options.ImageListSelection = cp.CopyAllImages
Expand Down
71 changes: 37 additions & 34 deletions cmd/buildah/push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,29 @@ import (
)

type pushOptions struct {
all bool
authfile string
blobCache string
certDir string
creds string
digestfile string
disableCompression bool
format string
compressionFormat string
compressionLevel int
retry int
retryDelay string
rm bool
quiet bool
removeSignatures bool
signaturePolicy string
signBy string
tlsVerify bool
encryptionKeys []string
encryptLayers []int
insecure bool
addCompression []string
all bool
authfile string
blobCache string
certDir string
creds string
digestfile string
disableCompression bool
format string
compressionFormat string
compressionLevel int
forceCompressionFormat bool
retry int
retryDelay string
rm bool
quiet bool
removeSignatures bool
signaturePolicy string
signBy string
tlsVerify bool
encryptionKeys []string
encryptLayers []int
insecure bool
addCompression []string
}

func init() {
Expand Down Expand Up @@ -86,6 +87,7 @@ func init() {
flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
flags.StringVar(&opts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting image to the file")
flags.BoolVarP(&opts.disableCompression, "disable-compression", "D", false, "don't compress layers")
flags.BoolVarP(&opts.forceCompressionFormat, "force-compression", "", false, "force re-compressing layers with specified format while pushing")
flags.StringVarP(&opts.format, "format", "f", "", "manifest type (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)")
flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use")
flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use")
Expand Down Expand Up @@ -199,18 +201,19 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
}

options := buildah.PushOptions{
Compression: compress,
ManifestType: manifestType,
SignaturePolicyPath: iopts.signaturePolicy,
Store: store,
SystemContext: systemContext,
BlobDirectory: iopts.blobCache,
RemoveSignatures: iopts.removeSignatures,
SignBy: iopts.signBy,
MaxRetries: iopts.retry,
RetryDelay: pullPushRetryDelay,
OciEncryptConfig: encConfig,
OciEncryptLayers: encLayers,
Compression: compress,
ManifestType: manifestType,
SignaturePolicyPath: iopts.signaturePolicy,
Store: store,
SystemContext: systemContext,
BlobDirectory: iopts.blobCache,
RemoveSignatures: iopts.removeSignatures,
SignBy: iopts.signBy,
MaxRetries: iopts.retry,
RetryDelay: pullPushRetryDelay,
OciEncryptConfig: encConfig,
OciEncryptLayers: encLayers,
ForceCompressionFormat: iopts.forceCompressionFormat,
}
if !iopts.quiet {
options.ReportWriter = os.Stderr
Expand Down
5 changes: 5 additions & 0 deletions push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ type PushOptions struct {
CompressionFormat *compression.Algorithm
// CompressionLevel specifies what compression level is used
CompressionLevel *int
// ForceCompressionFormat ensures that the compression algorithm set in
// CompressionFormat is used exclusively, and blobs of other compression
// algorithms are not reused.
ForceCompressionFormat bool
}

// Push copies the contents of the image to a new location.
Expand All @@ -110,6 +114,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options
libimageOptions.OciEncryptLayers = options.OciEncryptLayers
libimageOptions.CompressionFormat = options.CompressionFormat
libimageOptions.CompressionLevel = options.CompressionLevel
libimageOptions.ForceCompressionFormat = options.ForceCompressionFormat
libimageOptions.PolicyAllowStorage = true

if options.Quiet {
Expand Down
66 changes: 66 additions & 0 deletions tests/bud.bats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,72 @@ _EOF
validate_instance_compression "3" "$list" "arm64" "zstd"
}

@test "bud: build manifest list with --add-compression zstd, --compression and --force-compression" {
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
mkdir -p $contextdir

cat > $contextdir/Dockerfile1 << _EOF
FROM alpine
_EOF

start_registry
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1
run_buildah build $WITH_POLICY_JSON -t image2 --platform linux/arm64 -f $contextdir/Dockerfile1

run_buildah manifest create foo
run_buildah manifest add foo image1
run_buildah manifest add foo image2

run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list

run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
list="$output"

validate_instance_compression "0" "$list" "amd64" "gzip"
validate_instance_compression "1" "$list" "arm64" "gzip"
validate_instance_compression "2" "$list" "amd64" "zstd"
validate_instance_compression "3" "$list" "arm64" "zstd"

# Pushing again should keep every thing intact if original compression is `gzip` and `--force-compression` is specified
run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --compression-format gzip --force-compression --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list

run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
list="$output"

validate_instance_compression "0" "$list" "amd64" "gzip"
validate_instance_compression "1" "$list" "arm64" "gzip"
validate_instance_compression "2" "$list" "amd64" "zstd"
validate_instance_compression "3" "$list" "arm64" "zstd"
}

@test "bud: build push with --force-compression" {
skip_if_no_podman
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
mkdir -p $contextdir

cat > $contextdir/Dockerfile1 << _EOF
FROM alpine
_EOF

start_registry
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1

run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format gzip image1 docker://localhost:${REGISTRY_PORT}/image
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
# layers should have no trace of zstd since push was with --compression-format gzip
assert "$output" !~ "zstd" "zstd found in layers where push was with --compression-format gzip"
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd image1 docker://localhost:${REGISTRY_PORT}/image
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
# layers should have no trace of zstd since push is without --force-compression
assert "$output" !~ "zstd" "zstd found even though push was without --force-compression"
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd --force-compression image1 docker://localhost:${REGISTRY_PORT}/image
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
# layers should container `zstd`
expect_output --substring "zstd" "layers must contain zstd compression"
}

@test "bud with --dns* flags" {
_prefetch alpine

Expand Down

0 comments on commit b79361c

Please sign in to comment.