Use lzmadec to support streaming from 7z archives

RELEASE_NOTES=[ENHANCEMENT] Support reading from 7z archives

Signed-off-by: Dominik Schulz <[email protected]>

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/kjk/lzmadec v0.0.0-20210713164611-19ac3ee91a71 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect

go.sum

@@ -57,6 +57,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/kjk/lzmadec v0.0.0-20210713164611-19ac3ee91a71 h1:TYp9Fj0apeZMWentXRaFM6B0ixdFefrlgY8n8XYEz1s=
+github.com/kjk/lzmadec v0.0.0-20210713164611-19ac3ee91a71/go.mod h1:2zRkQCuw/eK6cqkYAeNqyBU7JKa2Gcq40BZ9GSJbmfE=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=

hibp.go

@@ -56,6 +56,16 @@ func (s *hibp) CheckAPI(ctx context.Context, force bool) error {
 func (s *hibp) CheckDump(ctx context.Context, force bool, dumps []string) error {
 	fmt.Println("Using the HIBPv2 dumps is very expensive. If you can condone leaking a few bits of entropy per secret you should probably use the '--api' flag.")
 
+	if len(dumps) < 1 {
+		return fmt.Errorf("need a least one dump file")
+	}
+
+	// New also checks if there is at least one valid dump file given
+	scanner, err := hibpdump.New(dumps...)
+	if err != nil {
+		return fmt.Errorf("failed to create new HIBP Dump scanner: %w", err)
+	}
+
 	if !force && !termio.AskForConfirmation(ctx, fmt.Sprintf("This command is checking all your secrets against the haveibeenpwned.com hashes in %+v.\nYou will be asked to unlock all your secrets!\nDo you want to continue?", dumps)) {
 		return fmt.Errorf("user aborted")
 	}
@@ -65,10 +75,7 @@ func (s *hibp) CheckDump(ctx context.Context, force bool, dumps []string) error
 		return err
 	}
 
-	scanner, err := hibpdump.New(dumps...)
-	if err != nil {
-		return fmt.Errorf("failed to create new HIBP Dump scanner: %w", err)
-	}
+	fmt.Println("Checking hashes against the provided dumps. This will take a while.")
 
 	matchedSums := scanner.LookupBatch(ctx, sortedShaSums)
 	debug.Log("In: %+v - Out: %+v", sortedShaSums, matchedSums)

pkg/hibp/dump/scanner.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/gopasspw/gopass/pkg/debug"
 	"github.com/gopasspw/gopass/pkg/fsutil"
+	"github.com/kjk/lzmadec"
 )
 
 // Scanner is a HIBP dump scanner.
@@ -106,9 +107,9 @@ func isSorted(fn string) bool {
 	defer func() {
 		_ = fh.Close()
 	}()
-	rdr = fh
 
-	if strings.HasSuffix(fn, ".gz") {
+	switch {
+	case strings.HasSuffix(fn, ".gz"):
 		gzr, err := gzip.NewReader(fh)
 		if err != nil {
 			return false
@@ -117,6 +118,24 @@ func isSorted(fn string) bool {
 			_ = gzr.Close()
 		}()
 		rdr = gzr
+	case strings.HasSuffix(fn, ".7z"):
+		arc, err := lzmadec.NewArchive(fn)
+		if err != nil {
+			return false
+		}
+		if len(arc.Entries) < 1 {
+			return false
+		}
+		rzr, err := arc.GetFileReader(arc.Entries[0].Path)
+		if err != nil {
+			return false
+		}
+		defer func() {
+			_ = rzr.Close()
+		}()
+		rdr = rzr
+	default:
+		rdr = fh
 	}
 
 	lineNo := 0
@@ -152,19 +171,43 @@ func (s *Scanner) scanSortedFile(ctx context.Context, fn string, in []string, re
 	defer func() {
 		_ = fh.Close()
 	}()
-	rdr = fh
 
-	if strings.HasSuffix(fn, ".gz") {
+	switch {
+	case strings.HasSuffix(fn, ".gz"):
 		gzr, err := gzip.NewReader(fh)
 		if err != nil {
-			fmt.Printf("Failed to open the file %s: %s", fn, err)
+			fmt.Printf("Failed to open the file with gzip %s: %s", fn, err)
 
 			return
 		}
 		defer func() {
 			_ = gzr.Close()
 		}()
 		rdr = gzr
+	case strings.HasSuffix(fn, ".7z"):
+		arc, err := lzmadec.NewArchive(fn)
+		if err != nil {
+			fmt.Printf("Failed to open the file with 7z %s: %s", fn, err)
+
+			return
+		}
+		if len(arc.Entries) < 1 {
+			fmt.Printf("7z archive %s contains no entries", fn)
+
+			return
+		}
+		rzr, err := arc.GetFileReader(arc.Entries[0].Path)
+		if err != nil {
+			fmt.Printf("Failed open %s in %s for reading: %s", arc.Entries[0].Path, fn, err)
+
+			return
+		}
+		defer func() {
+			_ = rzr.Close()
+		}()
+		rdr = rzr
+	default:
+		rdr = fh
 	}
 
 	debug.Log("Checking file %s ...\n", fn)