From d81e79540b300fb481c0c42bdf03818cfe91823d Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 9 Nov 2021 09:56:04 +0100 Subject: [PATCH] update to github.com/mitchellh/hashstructure v2.0.2 Signed-off-by: CrazyMax --- executor/oci/spec.go | 4 +- go.mod | 2 +- go.sum | 2 + solver/llbsolver/bridge.go | 4 +- .../mitchellh/hashstructure/{ => v2}/LICENSE | 0 .../hashstructure/{ => v2}/README.md | 21 ++- .../mitchellh/hashstructure/v2/errors.go | 22 +++ .../hashstructure/{ => v2}/hashstructure.go | 164 +++++++++++++++--- .../hashstructure/{ => v2}/include.go | 7 + vendor/modules.txt | 6 +- 10 files changed, 199 insertions(+), 33 deletions(-) rename vendor/github.com/mitchellh/hashstructure/{ => v2}/LICENSE (100%) rename vendor/github.com/mitchellh/hashstructure/{ => v2}/README.md (65%) create mode 100644 vendor/github.com/mitchellh/hashstructure/v2/errors.go rename vendor/github.com/mitchellh/hashstructure/{ => v2}/hashstructure.go (63%) rename vendor/github.com/mitchellh/hashstructure/{ => v2}/include.go (68%) diff --git a/executor/oci/spec.go b/executor/oci/spec.go index 7ea872fdcd12..8ca9a4514c7b 100644 --- a/executor/oci/spec.go +++ b/executor/oci/spec.go @@ -13,7 +13,7 @@ import ( "github.com/containerd/containerd/oci" "github.com/containerd/continuity/fs" "github.com/docker/docker/pkg/idtools" - "github.com/mitchellh/hashstructure" + "github.com/mitchellh/hashstructure/v2" "github.com/moby/buildkit/executor" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/util/network" @@ -200,7 +200,7 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error) if s.m == nil { s.m = map[uint64]mountRef{} } - h, err := hashstructure.Hash(m, nil) + h, err := hashstructure.Hash(m, hashstructure.FormatV2, nil) if err != nil { return mount.Mount{}, nil } diff --git a/go.mod b/go.mod index b724c1ec366a..7bd89fb90424 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/hashicorp/golang-lru v0.5.3 github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee // indirect github.com/klauspost/compress v1.13.5 - github.com/mitchellh/hashstructure v1.0.0 + github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/moby/locker v1.0.1 github.com/moby/sys/mount v0.3.0 // indirect github.com/moby/sys/signal v0.6.0 diff --git a/go.sum b/go.sum index e76f32a50e30..5ffa66e4e974 100644 --- a/go.sum +++ b/go.sum @@ -864,6 +864,8 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index c14cc99ce0d4..26ca7d2ca096 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -8,7 +8,7 @@ import ( "time" "github.com/containerd/containerd/platforms" - "github.com/mitchellh/hashstructure" + "github.com/mitchellh/hashstructure/v2" "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend" @@ -345,7 +345,7 @@ func cmKey(im gw.CacheOptionsEntry) (string, error) { if im.Type == "registry" && im.Attrs["ref"] != "" { return im.Attrs["ref"], nil } - i, err := hashstructure.Hash(im, nil) + i, err := hashstructure.Hash(im, hashstructure.FormatV2, nil) if err != nil { return "", err } diff --git a/vendor/github.com/mitchellh/hashstructure/LICENSE b/vendor/github.com/mitchellh/hashstructure/v2/LICENSE similarity index 100% rename from vendor/github.com/mitchellh/hashstructure/LICENSE rename to vendor/github.com/mitchellh/hashstructure/v2/LICENSE diff --git a/vendor/github.com/mitchellh/hashstructure/README.md b/vendor/github.com/mitchellh/hashstructure/v2/README.md similarity index 65% rename from vendor/github.com/mitchellh/hashstructure/README.md rename to vendor/github.com/mitchellh/hashstructure/v2/README.md index 28ce45a3e181..21f36be193e1 100644 --- a/vendor/github.com/mitchellh/hashstructure/README.md +++ b/vendor/github.com/mitchellh/hashstructure/v2/README.md @@ -17,20 +17,31 @@ sending data across the network, caching values locally (de-dup), and so on. doesn't affect the hash code but the field itself is still taken into account to create the hash value. - * Optionally specify a custom hash function to optimize for speed, collision + * Optionally, specify a custom hash function to optimize for speed, collision avoidance for your data set, etc. - - * Optionally hash the output of `.String()` on structs that implement fmt.Stringer, + + * Optionally, hash the output of `.String()` on structs that implement fmt.Stringer, allowing effective hashing of time.Time + * Optionally, override the hashing process by implementing `Hashable`. + ## Installation Standard `go get`: ``` -$ go get github.com/mitchellh/hashstructure +$ go get github.com/mitchellh/hashstructure/v2 ``` +**Note on v2:** It is highly recommended you use the "v2" release since this +fixes some significant hash collisions issues from v1. In practice, we used +v1 for many years in real projects at HashiCorp and never had issues, but it +is highly dependent on the shape of the data you're hashing and how you use +those hashes. + +When using v2+, you can still generate weaker v1 hashes by using the +`FormatV1` format when calling `Hash`. + ## Usage & Example For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure). @@ -54,7 +65,7 @@ v := ComplexStruct{ }, } -hash, err := hashstructure.Hash(v, nil) +hash, err := hashstructure.Hash(v, hashstructure.FormatV2, nil) if err != nil { panic(err) } diff --git a/vendor/github.com/mitchellh/hashstructure/v2/errors.go b/vendor/github.com/mitchellh/hashstructure/v2/errors.go new file mode 100644 index 000000000000..44b8951478ec --- /dev/null +++ b/vendor/github.com/mitchellh/hashstructure/v2/errors.go @@ -0,0 +1,22 @@ +package hashstructure + +import ( + "fmt" +) + +// ErrNotStringer is returned when there's an error with hash:"string" +type ErrNotStringer struct { + Field string +} + +// Error implements error for ErrNotStringer +func (ens *ErrNotStringer) Error() string { + return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field) +} + +// ErrFormat is returned when an invalid format is given to the Hash function. +type ErrFormat struct{} + +func (*ErrFormat) Error() string { + return "format must be one of the defined Format values in the hashstructure library" +} diff --git a/vendor/github.com/mitchellh/hashstructure/hashstructure.go b/vendor/github.com/mitchellh/hashstructure/v2/hashstructure.go similarity index 63% rename from vendor/github.com/mitchellh/hashstructure/hashstructure.go rename to vendor/github.com/mitchellh/hashstructure/v2/hashstructure.go index ea13a1583c3b..3dc0eb74e07f 100644 --- a/vendor/github.com/mitchellh/hashstructure/hashstructure.go +++ b/vendor/github.com/mitchellh/hashstructure/v2/hashstructure.go @@ -6,18 +6,9 @@ import ( "hash" "hash/fnv" "reflect" + "time" ) -// ErrNotStringer is returned when there's an error with hash:"string" -type ErrNotStringer struct { - Field string -} - -// Error implements error for ErrNotStringer -func (ens *ErrNotStringer) Error() string { - return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field) -} - // HashOptions are options that are available for hashing. type HashOptions struct { // Hasher is the hash function to use. If this isn't set, it will @@ -31,8 +22,42 @@ type HashOptions struct { // ZeroNil is flag determining if nil pointer should be treated equal // to a zero value of pointed type. By default this is false. ZeroNil bool + + // IgnoreZeroValue is determining if zero value fields should be + // ignored for hash calculation. + IgnoreZeroValue bool + + // SlicesAsSets assumes that a `set` tag is always present for slices. + // Default is false (in which case the tag is used instead) + SlicesAsSets bool + + // UseStringer will attempt to use fmt.Stringer always. If the struct + // doesn't implement fmt.Stringer, it'll fall back to trying usual tricks. + // If this is true, and the "string" tag is also set, the tag takes + // precedence (meaning that if the type doesn't implement fmt.Stringer, we + // panic) + UseStringer bool } +// Format specifies the hashing process used. Different formats typically +// generate different hashes for the same value and have different properties. +type Format uint + +const ( + // To disallow the zero value + formatInvalid Format = iota + + // FormatV1 is the format used in v1.x of this library. This has the + // downsides noted in issue #18 but allows simultaneous v1/v2 usage. + FormatV1 + + // FormatV2 is the current recommended format and fixes the issues + // noted in FormatV1. + FormatV2 + + formatMax // so we can easily find the end +) + // Hash returns the hash value of an arbitrary value. // // If opts is nil, then default options will be used. See HashOptions @@ -40,6 +65,11 @@ type HashOptions struct { // concurrently. None of the values within a *HashOptions struct are // safe to read/write while hashing is being done. // +// The "format" is required and must be one of the format values defined +// by this library. You should probably just use "FormatV2". This allows +// generated hashes uses alternate logic to maintain compatibility with +// older versions. +// // Notes on the value: // // * Unexported fields on structs are ignored and do not affect the @@ -65,7 +95,12 @@ type HashOptions struct { // * "string" - The field will be hashed as a string, only works when the // field implements fmt.Stringer // -func Hash(v interface{}, opts *HashOptions) (uint64, error) { +func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) { + // Validate our format + if format <= formatInvalid || format >= formatMax { + return 0, &ErrFormat{} + } + // Create default options if opts == nil { opts = &HashOptions{} @@ -82,17 +117,25 @@ func Hash(v interface{}, opts *HashOptions) (uint64, error) { // Create our walker and walk the structure w := &walker{ - h: opts.Hasher, - tag: opts.TagName, - zeronil: opts.ZeroNil, + format: format, + h: opts.Hasher, + tag: opts.TagName, + zeronil: opts.ZeroNil, + ignorezerovalue: opts.IgnoreZeroValue, + sets: opts.SlicesAsSets, + stringer: opts.UseStringer, } return w.visit(reflect.ValueOf(v), nil) } type walker struct { - h hash.Hash64 - tag string - zeronil bool + format Format + h hash.Hash64 + tag string + zeronil bool + ignorezerovalue bool + sets bool + stringer bool } type visitOpts struct { @@ -104,6 +147,8 @@ type visitOpts struct { StructField string } +var timeType = reflect.TypeOf(time.Time{}) + func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { t := reflect.TypeOf(0) @@ -159,6 +204,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { return w.h.Sum64(), err } + switch v.Type() { + case timeType: + w.h.Reset() + b, err := v.Interface().(time.Time).MarshalBinary() + if err != nil { + return 0, err + } + + err = binary.Write(w.h, binary.LittleEndian, b) + return w.h.Sum64(), err + } + switch k { case reflect.Array: var h uint64 @@ -211,6 +268,11 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { h = hashUpdateUnordered(h, fieldHash) } + if w.format != FormatV1 { + // Important: read the docs for hashFinishUnordered + h = hashFinishUnordered(w.h, h) + } + return h, nil case reflect.Struct: @@ -220,6 +282,24 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { include = impl } + if impl, ok := parent.(Hashable); ok { + return impl.Hash() + } + + // If we can address this value, check if the pointer value + // implements our interfaces and use that if so. + if v.CanAddr() { + vptr := v.Addr() + parentptr := vptr.Interface() + if impl, ok := parentptr.(Includable); ok { + include = impl + } + + if impl, ok := parentptr.(Hashable); ok { + return impl.Hash() + } + } + t := v.Type() h, err := w.visit(reflect.ValueOf(t.Name()), nil) if err != nil { @@ -242,11 +322,19 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { continue } + if w.ignorezerovalue { + if innerV.IsZero() { + continue + } + } + // if string is set, use the string value - if tag == "string" { + if tag == "string" || w.stringer { if impl, ok := innerV.Interface().(fmt.Stringer); ok { innerV = reflect.ValueOf(impl.String()) - } else { + } else if tag == "string" { + // We only show this error if the tag explicitly + // requests a stringer. return 0, &ErrNotStringer{ Field: v.Type().Field(i).Name, } @@ -286,6 +374,11 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { fieldHash := hashUpdateOrdered(w.h, kh, vh) h = hashUpdateUnordered(h, fieldHash) } + + if w.format != FormatV1 { + // Important: read the docs for hashFinishUnordered + h = hashFinishUnordered(w.h, h) + } } return h, nil @@ -306,13 +399,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { return 0, err } - if set { + if set || w.sets { h = hashUpdateUnordered(h, current) } else { h = hashUpdateOrdered(w.h, h, current) } } + if set && w.format != FormatV1 { + // Important: read the docs for hashFinishUnordered + h = hashFinishUnordered(w.h, h) + } + return h, nil case reflect.String: @@ -349,6 +447,32 @@ func hashUpdateUnordered(a, b uint64) uint64 { return a ^ b } +// After mixing a group of unique hashes with hashUpdateUnordered, it's always +// necessary to call hashFinishUnordered. Why? Because hashUpdateUnordered +// is a simple XOR, and calling hashUpdateUnordered on hashes produced by +// hashUpdateUnordered can effectively cancel out a previous change to the hash +// result if the same hash value appears later on. For example, consider: +// +// hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) = +// H("A") ^ H("B")) ^ (H("A") ^ H("C")) = +// (H("A") ^ H("A")) ^ (H("B") ^ H(C)) = +// H(B) ^ H(C) = +// hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C")) +// +// hashFinishUnordered "hardens" the result, so that encountering partially +// overlapping input data later on in a different context won't cancel out. +func hashFinishUnordered(h hash.Hash64, a uint64) uint64 { + h.Reset() + + // We just panic if the writes fail + e1 := binary.Write(h, binary.LittleEndian, a) + if e1 != nil { + panic(e1) + } + + return h.Sum64() +} + // visitFlag is used as a bitmask for affecting visit behavior type visitFlag uint diff --git a/vendor/github.com/mitchellh/hashstructure/include.go b/vendor/github.com/mitchellh/hashstructure/v2/include.go similarity index 68% rename from vendor/github.com/mitchellh/hashstructure/include.go rename to vendor/github.com/mitchellh/hashstructure/v2/include.go index b6289c0bee71..702d35415d4a 100644 --- a/vendor/github.com/mitchellh/hashstructure/include.go +++ b/vendor/github.com/mitchellh/hashstructure/v2/include.go @@ -13,3 +13,10 @@ type Includable interface { type IncludableMap interface { HashIncludeMap(field string, k, v interface{}) (bool, error) } + +// Hashable is an interface that can optionally be implemented by a struct +// to override the hash value. This value will override the hash value for +// the entire struct. Entries in the struct will not be hashed. +type Hashable interface { + Hash() (uint64, error) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 566fc619eea2..ac30b0a8da18 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -365,9 +365,9 @@ github.com/klauspost/compress/zstd/internal/xxhash # github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 ## explicit; go 1.9 github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/mitchellh/hashstructure v1.0.0 -## explicit -github.com/mitchellh/hashstructure +# github.com/mitchellh/hashstructure/v2 v2.0.2 +## explicit; go 1.14 +github.com/mitchellh/hashstructure/v2 # github.com/moby/locker v1.0.1 ## explicit; go 1.13 github.com/moby/locker