Merge branch 'main' into fix-konflux-oom

bib/cmd/bootc-image-builder/cloud.go

@@ -1,35 +1,22 @@
 package main
 
 import (
+	"fmt"
+	"io"
+	"os"
+
 	"github.com/cheggaaa/pb/v3"
-	"github.com/osbuild/bootc-image-builder/bib/internal/uploader"
-	"github.com/osbuild/images/pkg/cloud/awscloud"
 	"github.com/spf13/pflag"
+
+	"github.com/osbuild/images/pkg/cloud"
 )
 
-func uploadAMI(path, targetArch string, flags *pflag.FlagSet) error {
-	region, err := flags.GetString("aws-region")
-	if err != nil {
-		return err
-	}
-	bucketName, err := flags.GetString("aws-bucket")
-	if err != nil {
-		return err
-	}
-	imageName, err := flags.GetString("aws-ami-name")
-	if err != nil {
-		return err
-	}
+func upload(uploader cloud.Uploader, path string, flags *pflag.FlagSet) error {
 	progress, err := flags.GetString("progress")
 	if err != nil {
 		return err
 	}
 
-	client, err := awscloud.NewDefault(region)
-	if err != nil {
-		return err
-	}
-
 	// TODO: extract this as a helper once we add "uploadAzure" or
 	// similar. Eventually we may provide json progress here too.
 	var pbar *pb.ProgressBar
@@ -38,5 +25,25 @@ func uploadAMI(path, targetArch string, flags *pflag.FlagSet) error {
 		pbar = pb.New(0)
 	}
 
-	return uploader.UploadAndRegister(client, path, bucketName, imageName, targetArch, pbar)
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("cannot upload: %v", err)
+	}
+	defer file.Close()
+
+	var r io.Reader = file
+	if pbar != nil {
+		st, err := file.Stat()
+		if err != nil {
+			return err
+		}
+		pbar.SetTotal(st.Size())
+		pbar.Set(pb.Bytes, true)
+		pbar.SetWriter(osStdout)
+		r = pbar.NewProxyReader(file)
+		pbar.Start()
+		defer pbar.Finish()
+	}
+
+	return uploader.UploadAndRegister(r, osStderr)
 }

bib/cmd/bootc-image-builder/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -17,6 +18,7 @@ import (
 	"golang.org/x/exp/slices"
 
 	"github.com/osbuild/images/pkg/arch"
+	"github.com/osbuild/images/pkg/cloud"
 	"github.com/osbuild/images/pkg/cloud/awscloud"
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/dnfjson"
@@ -53,6 +55,9 @@ var distroDefPaths = []string{
 var (
 	osGetuid = os.Getuid
 	osGetgid = os.Getgid
+
+	osStdout = os.Stdout
+	osStderr = os.Stderr
 )
 
 // canChownInPath checks if the ownership of files can be set in a given path.
@@ -340,53 +345,36 @@ func cmdManifest(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func handleAWSFlags(cmd *cobra.Command) (upload bool, err error) {
+func handleAWSFlags(cmd *cobra.Command) (cloud.Uploader, error) {
 	imgTypes, _ := cmd.Flags().GetStringArray("type")
 	region, _ := cmd.Flags().GetString("aws-region")
 	if region == "" {
-		return false, nil
+		return nil, nil
 	}
 	bucketName, _ := cmd.Flags().GetString("aws-bucket")
+	imageName, _ := cmd.Flags().GetString("aws-ami-name")
+	targetArch, _ := cmd.Flags().GetString("target-arch")
 
 	if !slices.Contains(imgTypes, "ami") {
-		return false, fmt.Errorf("aws flags set for non-ami image type (type is set to %s)", strings.Join(imgTypes, ","))
+		return nil, fmt.Errorf("aws flags set for non-ami image type (type is set to %s)", strings.Join(imgTypes, ","))
 	}
 
 	// check as many permission prerequisites as possible before starting
-	client, err := awscloud.NewDefault(region)
-	if err != nil {
-		return false, err
-	}
-
-	logrus.Info("Checking AWS region access...")
-	regions, err := client.Regions()
-	if err != nil {
-		return false, fmt.Errorf("retrieving AWS regions for '%s' failed: %w", region, err)
-	}
-
-	if !slices.Contains(regions, region) {
-		return false, fmt.Errorf("given AWS region '%s' not found", region)
+	uploaderOpts := &awscloud.UploaderOptions{
+		TargetArch: targetArch,
 	}
-
-	logrus.Info("Checking AWS bucket...")
-	buckets, err := client.Buckets()
+	uploader, err := awscloud.NewUploader(region, bucketName, imageName, uploaderOpts)
 	if err != nil {
-		return false, fmt.Errorf("retrieving AWS list of buckets failed: %w", err)
-	}
-	if !slices.Contains(buckets, bucketName) {
-		return false, fmt.Errorf("bucket '%s' not found in the given AWS account", bucketName)
+		return nil, err
 	}
-
-	logrus.Info("Checking AWS bucket permissions...")
-	writePermission, err := client.CheckBucketPermission(bucketName, awscloud.S3PermissionWrite)
-	if err != nil {
-		return false, err
+	status := io.Discard
+	if logrus.GetLevel() >= logrus.InfoLevel {
+		status = os.Stderr
 	}
-	if !writePermission {
-		return false, fmt.Errorf("you don't have write permissions to bucket '%s' with the given AWS account", bucketName)
+	if err := uploader.Check(status); err != nil {
+		return nil, err
 	}
-	logrus.Info("Upload conditions met.")
-	return true, nil
+	return uploader, nil
 }
 
 func cmdBuild(cmd *cobra.Command, args []string) error {
@@ -415,7 +403,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("cannot setup build dir: %w", err)
 	}
 
-	upload, err := handleAWSFlags(cmd)
+	uploader, err := handleAWSFlags(cmd)
 	if err != nil {
 		return fmt.Errorf("cannot handle AWS setup: %w", err)
 	}
@@ -478,7 +466,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
 	}
 
 	pbar.SetMessagef("Build complete!")
-	if upload {
+	if uploader != nil {
 		// XXX: pass our own progress.ProgressBar here
 		// *for now* just stop our own progress and let the uploadAMI
 		// progress take over - but we really need to fix this in a
@@ -488,7 +476,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
 			switch imgType {
 			case "ami":
 				diskpath := filepath.Join(outputDir, exports[idx], "disk.raw")
-				if err := uploadAMI(diskpath, targetArch, cmd.Flags()); err != nil {
+				if err := upload(uploader, diskpath, cmd.Flags()); err != nil {
 					return fmt.Errorf("cannot upload AMI: %w", err)
 				}
 			default:

bib/cmd/upload/main.go

@@ -7,7 +7,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/osbuild/bootc-image-builder/bib/internal/uploader"
 	"github.com/osbuild/images/pkg/cloud/awscloud"
 )
 
@@ -26,16 +25,24 @@ func uploadAMI(cmd *cobra.Command, args []string) {
 
 	region, err := flags.GetString("region")
 	check(err)
-	client, err := awscloud.NewDefault(region)
-	check(err)
 	bucketName, err := flags.GetString("bucket")
 	check(err)
 	imageName, err := flags.GetString("ami-name")
 	check(err)
 	targetArch, err := flags.GetString("target-arch")
 	check(err)
 
-	check(uploader.UploadAndRegister(client, filename, bucketName, imageName, targetArch, nil))
+	opts := &awscloud.UploaderOptions{
+		TargetArch: targetArch,
+	}
+	uploader, err := awscloud.NewUploader(region, bucketName, imageName, opts)
+	check(err)
+
+	f, err := os.Open(filename)
+	check(err)
+	defer f.Close()
+
+	check(uploader.UploadAndRegister(f, os.Stderr))
 }
 
 func setupCLI() *cobra.Command {

bib/go.mod

@@ -4,18 +4,16 @@ go 1.22.6
 
 require (
 	github.com/BurntSushi/toml v1.4.0
-	github.com/aws/aws-sdk-go v1.55.6
 	github.com/cheggaaa/pb/v3 v3.1.6
-	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-version v1.7.0
 	github.com/mattn/go-isatty v0.0.20
 	github.com/osbuild/images v0.116.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
-	github.com/spf13/pflag v1.0.5
+	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.10.0
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
-	golang.org/x/sys v0.29.0
+	golang.org/x/sys v0.30.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -26,6 +24,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/aws/aws-sdk-go v1.55.6 // indirect
 	github.com/containerd/cgroups/v3 v3.0.3 // indirect
 	github.com/containerd/errdefs v0.3.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
@@ -66,6 +65,7 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-containerregistry v0.20.2 // indirect
 	github.com/google/go-intervals v0.0.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect

bib/go.sum

@@ -274,8 +274,9 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -372,8 +373,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
 golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=