From e4bf412f2b00530ed0e832d3d7c257bfd97948be Mon Sep 17 00:00:00 2001 From: Sven Gregori Date: Thu, 9 Jan 2025 02:31:39 +0200 Subject: [PATCH 1/3] fix: fetch Dart package versions from sdk entries Packages that are provided by an SDK, mainly Flutter, will have their version set to 0.0.0 in Dart's pubspec.lock file. Their actual version is linked to that SDK, which is defined either as a version range or a minimum supported version, rather than an explicit, single version. The pubspec.lock file has a dedicated section to define those SDK version range constraints, which is already stored internally when parsing the file itself. The solution now is to look up such a package's SDK name, retrieve the defined version range / lower version boundary, and set the minimum supported version as the package's new version. Signed-off-by: Sven Gregori --- syft/pkg/cataloger/dart/parse_pubspec_lock.go | 86 +++++++++- .../cataloger/dart/parse_pubspec_lock_test.go | 161 +++++++++++++++++- .../test-fixtures/invalid-sdk/pubspec.lock | 8 + .../test-fixtures/missing-sdk/pubspec.lock | 6 + .../cataloger/dart/test-fixtures/pubspec.lock | 1 + 5 files changed, 258 insertions(+), 4 deletions(-) create mode 100644 syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock create mode 100644 syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock diff --git a/syft/pkg/cataloger/dart/parse_pubspec_lock.go b/syft/pkg/cataloger/dart/parse_pubspec_lock.go index ebf41f822da..86a950bb3fe 100644 --- a/syft/pkg/cataloger/dart/parse_pubspec_lock.go +++ b/syft/pkg/cataloger/dart/parse_pubspec_lock.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "net/url" + "regexp" "sort" "gopkg.in/yaml.v3" + "github.com/Masterminds/semver" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" @@ -68,8 +70,28 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment } var names []string - for name := range p.Packages { + for name, pkg := range p.Packages { names = append(names, name) + + if pkg.Source == "sdk" && pkg.Version == "0.0.0" { + // Packages that are delivered as part of an SDK (e.g. Flutter) have their + // version set to "0.0.0" in the package definition. The actual version + // should refer to the SDK version, which is defined in a dedicated section + // in the pubspec.lock file and uses a version range constraint. + // + // If such a package is detected, look up the version range constraint of + // its matching SDK, and set the minimum supported version as its new version. + sdkName := pkg.Description.Name + sdkVersion, err := p.getSdkVersion(sdkName) + + if err != nil { + log.Infof("Failed to resolve %s SDK version for package %s: %v", sdkName, name, err) + } else { + log.Debugf("Resolved %s SDK version for package %s to %s", sdkName, name, sdkVersion) + pkg.Version = sdkVersion + p.Packages[name] = pkg + } + } } // always ensure there is a stable ordering of packages @@ -89,6 +111,68 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages") } +// Look up the version range constraint for a given sdk name, if found, +// and return its lowest supported version matching that constraint. +// +// The sdks and their constraints are defined in the pubspec.lock file, e.g. +// +// sdks: +// dart: ">=2.12.0 <3.0.0" +// flutter: ">=3.24.5" +// +// and stored in the pubspecLock.Sdks map during parsing. +// +// Example based on the data above: +// +// getSdkVersion("dart") -> "2.12.0" +// getSdkVersion("flutter") -> "3.24.5" +// getSdkVersion("undefined") -> error +func (psl *pubspecLock) getSdkVersion(sdk string) (string, error) { + constraint, found := psl.Sdks[sdk] + + if !found { + return "", fmt.Errorf("cannot find %s SDK", sdk) + } + + return parseMinimumSdkVersion(constraint) +} + +// Parse a given version range constraint and return its lowest supported version. +// +// This is intended for packages that are part of an SDK (e.g. Flutter) and don't +// have an explicit version string set. This will take the given constraint +// parameter, ensure it's a valid constraint string, and return the lowest version +// within that constraint range. +// +// Examples: +// +// parseMinimumSdkVersion("^1.2.3") -> "1.2.3" +// parseMinimumSdkVersion(">=1.2.3") -> "1.2.3" +// parseMinimumSdkVersion(">=1.2.3 <2.0.0") -> "1.2.3" +// parseMinimumSdkVersion("1.2.3") -> error +// +// see https://dart.dev/tools/pub/dependencies#version-constraints for the +// constraint format used in Dart SDK defintions. +func parseMinimumSdkVersion(constraint string) (string, error) { + // Match strings that + // 1. start with either "^" or ">=" (Dart SDK constraints only use those two) + // 2. followed by a valid semantic version, matched as "version" named subexpression + // 3. followed by a space (if there's a range) or end of string (if there's only a lower boundary) + // |---1--||------------------2------------------||-3-| + re := regexp.MustCompile(`^(\^|>=)(?P` + semver.SemVerRegex + `)( |$)`) + + if !re.MatchString(constraint) { + return "", fmt.Errorf("unsupported or invalid constraint '%s'", constraint) + } + + // Read "version" subexpression (see 2. above) into version variable + version := []byte{} + matchIndex := re.FindStringSubmatchIndex(constraint) + version = re.ExpandString(version, "$version", constraint, matchIndex) + + return string(version), nil +} + func (p *pubspecLockPackage) getVcsURL() string { if p.Source == "git" { if p.Description.Path == "." { diff --git a/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go b/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go index 5aa5fc702b5..2309f55dd2e 100644 --- a/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go +++ b/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go @@ -7,6 +7,7 @@ import ( "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/stretchr/testify/assert" ) func TestParsePubspecLock(t *testing.T) { @@ -76,14 +77,14 @@ func TestParsePubspecLock(t *testing.T) { }, { Name: "flutter", - Version: "0.0.0", - PURL: "pkg:pub/flutter@0.0.0", + Version: "3.24.5", + PURL: "pkg:pub/flutter@3.24.5", Locations: fixtureLocationSet, Language: pkg.Dart, Type: pkg.DartPubPkg, Metadata: pkg.DartPubspecLockEntry{ Name: "flutter", - Version: "0.0.0", + Version: "3.24.5", }, }, { @@ -113,3 +114,157 @@ func Test_corruptPubspecLock(t *testing.T) { WithError(). TestParser(t, parsePubspecLock) } + +func Test_missingSdkEntryPubspecLock(t *testing.T) { + fixture := "test-fixtures/missing-sdk/pubspec.lock" + fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture)) + expected := []pkg.Package{ + { + Name: "flutter", + Version: "0.0.0", + PURL: "pkg:pub/flutter@0.0.0", + Locations: fixtureLocationSet, + Language: pkg.Dart, + Type: pkg.DartPubPkg, + Metadata: pkg.DartPubspecLockEntry{ + Name: "flutter", + Version: "0.0.0", + }, + }, + } + + // TODO: relationships are not under test + var expectedRelationships []artifact.Relationship + + pkgtest.TestFileParser(t, fixture, parsePubspecLock, expected, expectedRelationships) +} + +func Test_invalidSdkEntryPubspecLock(t *testing.T) { + fixture := "test-fixtures/invalid-sdk/pubspec.lock" + fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture)) + expected := []pkg.Package{ + { + Name: "flutter", + Version: "0.0.0", + PURL: "pkg:pub/flutter@0.0.0", + Locations: fixtureLocationSet, + Language: pkg.Dart, + Type: pkg.DartPubPkg, + Metadata: pkg.DartPubspecLockEntry{ + Name: "flutter", + Version: "0.0.0", + }, + }, + } + + // TODO: relationships are not under test + var expectedRelationships []artifact.Relationship + + pkgtest.TestFileParser(t, fixture, parsePubspecLock, expected, expectedRelationships) +} + +func Test_sdkVersionLookup(t *testing.T) { + psl := &pubspecLock{ + Sdks: make(map[string]string, 5), + } + + psl.Sdks["minVersionSdk"] = ">=0.1.2" + psl.Sdks["rangeVersionSdk"] = ">=1.2.3 <2.0.0" + psl.Sdks["caretVersionSdk"] = "^2.3.4" + psl.Sdks["emptyVersionSdk"] = "" + psl.Sdks["invalidVersionSdk"] = "not a constraint" + + var version string + var err error + + version, err = psl.getSdkVersion("minVersionSdk") + assert.NoError(t, err) + assert.Equal(t, "0.1.2", version) + + version, err = psl.getSdkVersion("rangeVersionSdk") + assert.NoError(t, err) + assert.Equal(t, "1.2.3", version) + + version, err = psl.getSdkVersion("caretVersionSdk") + assert.NoError(t, err) + assert.Equal(t, "2.3.4", version) + + version, err = psl.getSdkVersion("emptyVersionSdk") + assert.Error(t, err) + assert.Equal(t, "", version) + + version, err = psl.getSdkVersion("invalidVersionSdk") + assert.Error(t, err) + assert.Equal(t, "", version) + + version, err = psl.getSdkVersion("nonexistantSdk") + assert.Error(t, err) + assert.Equal(t, "", version) +} + +func Test_sdkVersionParser_valid(t *testing.T) { + var version string + var err error + + // map constraints to expected version + patterns := map[string]string{ + "^0.0.0": "0.0.0", + ">=0.0.0": "0.0.0", + "^1.23.4": "1.23.4", + ">=1.23.4": "1.23.4", + "^11.22.33": "11.22.33", + ">=11.22.33": "11.22.33", + "^123.123456.12345678": "123.123456.12345678", + ">=123.123456.12345678": "123.123456.12345678", + ">=1.2.3 <2.3.4": "1.2.3", + ">=1.2.3 random string": "1.2.3", + ">=1.2.3 >=0.1.2": "1.2.3", + "^1.2": "1.2", + ">=1.2": "1.2", + "^1.2.3-rc4": "1.2.3-rc4", + ">=1.2.3-rc4": "1.2.3-rc4", + "^2.34.5+hotfix6": "2.34.5+hotfix6", + ">=2.34.5+hotfix6": "2.34.5+hotfix6", + } + + for constraint, expected := range patterns { + version, err = parseMinimumSdkVersion(constraint) + assert.NoError(t, err) + assert.Equalf(t, expected, version, "constraint '%s", constraint) + } +} + +func Test_sdkVersionParser_invalid(t *testing.T) { + var version string + var err error + + patterns := []string{ + "", + "abc", + "^abc", + ">=abc", + "^a.b.c", + ">=a.b.c", + "1.2.34", + ">1.2.34", + "<=1.2.34", + "<1.2.34", + "^1.2.3.4", + ">=1.2.3.4", + "^1.x.0", + ">=1.x.0", + "^1x2x3", + ">=1x2x3", + "^1.-2.3", + ">=1.-2.3", + "abc <1.2.34", + "^2.3.45hotfix6", + ">=2.3.45hotfix6", + } + + for _, pattern := range patterns { + version, err = parseMinimumSdkVersion(pattern) + assert.Error(t, err) + assert.Equalf(t, "", version, "constraint '%s'", pattern) + } +} diff --git a/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock b/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock new file mode 100644 index 00000000000..6b3a71aa3f9 --- /dev/null +++ b/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock @@ -0,0 +1,8 @@ +packages: + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" +sdks: + flutter: "3.24.5" diff --git a/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock b/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock new file mode 100644 index 00000000000..ea3ed219f86 --- /dev/null +++ b/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock @@ -0,0 +1,6 @@ +packages: + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" diff --git a/syft/pkg/cataloger/dart/test-fixtures/pubspec.lock b/syft/pkg/cataloger/dart/test-fixtures/pubspec.lock index da464c79527..46014bdc05e 100644 --- a/syft/pkg/cataloger/dart/test-fixtures/pubspec.lock +++ b/syft/pkg/cataloger/dart/test-fixtures/pubspec.lock @@ -52,3 +52,4 @@ packages: version: "1.11.20" sdks: dart: ">=2.12.0 <3.0.0" + flutter: ">=3.24.5" From 57229ef1e48f4095832f7002550e6c59835f7447 Mon Sep 17 00:00:00 2001 From: Sven Gregori Date: Fri, 17 Jan 2025 02:05:51 +0200 Subject: [PATCH 2/3] Ignore Dart package if SDK version cannot be fetched Signed-off-by: Sven Gregori --- syft/pkg/cataloger/dart/parse_pubspec_lock.go | 10 ++++--- .../cataloger/dart/parse_pubspec_lock_test.go | 26 ++++++++++++------- .../test-fixtures/invalid-sdk/pubspec.lock | 7 +++++ .../test-fixtures/missing-sdk/pubspec.lock | 7 +++++ 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/syft/pkg/cataloger/dart/parse_pubspec_lock.go b/syft/pkg/cataloger/dart/parse_pubspec_lock.go index 86a950bb3fe..0970dd5126f 100644 --- a/syft/pkg/cataloger/dart/parse_pubspec_lock.go +++ b/syft/pkg/cataloger/dart/parse_pubspec_lock.go @@ -71,8 +71,6 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment var names []string for name, pkg := range p.Packages { - names = append(names, name) - if pkg.Source == "sdk" && pkg.Version == "0.0.0" { // Packages that are delivered as part of an SDK (e.g. Flutter) have their // version set to "0.0.0" in the package definition. The actual version @@ -85,13 +83,17 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment sdkVersion, err := p.getSdkVersion(sdkName) if err != nil { - log.Infof("Failed to resolve %s SDK version for package %s: %v", sdkName, name, err) + log.Tracef("failed to resolve %s SDK version for package %s: %v", sdkName, name, err) + continue + } else { - log.Debugf("Resolved %s SDK version for package %s to %s", sdkName, name, sdkVersion) + log.Tracef("resolved %s SDK version for package %s to %s", sdkName, name, sdkVersion) pkg.Version = sdkVersion p.Packages[name] = pkg } } + + names = append(names, name) } // always ensure there is a stable ordering of packages diff --git a/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go b/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go index 2309f55dd2e..c002bf2c023 100644 --- a/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go +++ b/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go @@ -118,17 +118,20 @@ func Test_corruptPubspecLock(t *testing.T) { func Test_missingSdkEntryPubspecLock(t *testing.T) { fixture := "test-fixtures/missing-sdk/pubspec.lock" fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture)) + + // SDK version is missing, so flutter version cannot be determined and + // is ignored, expecting args as only package in the list as a result. expected := []pkg.Package{ { - Name: "flutter", - Version: "0.0.0", - PURL: "pkg:pub/flutter@0.0.0", + Name: "args", + Version: "1.6.0", + PURL: "pkg:pub/args@1.6.0", Locations: fixtureLocationSet, Language: pkg.Dart, Type: pkg.DartPubPkg, Metadata: pkg.DartPubspecLockEntry{ - Name: "flutter", - Version: "0.0.0", + Name: "args", + Version: "1.6.0", }, }, } @@ -142,17 +145,20 @@ func Test_missingSdkEntryPubspecLock(t *testing.T) { func Test_invalidSdkEntryPubspecLock(t *testing.T) { fixture := "test-fixtures/invalid-sdk/pubspec.lock" fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture)) + + // SDK version is invalid, so flutter version cannot be determined and + // is ignored, expecting args as only package in the list as a result. expected := []pkg.Package{ { - Name: "flutter", - Version: "0.0.0", - PURL: "pkg:pub/flutter@0.0.0", + Name: "args", + Version: "1.6.0", + PURL: "pkg:pub/args@1.6.0", Locations: fixtureLocationSet, Language: pkg.Dart, Type: pkg.DartPubPkg, Metadata: pkg.DartPubspecLockEntry{ - Name: "flutter", - Version: "0.0.0", + Name: "args", + Version: "1.6.0", }, }, } diff --git a/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock b/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock index 6b3a71aa3f9..1743816651b 100644 --- a/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock +++ b/syft/pkg/cataloger/dart/test-fixtures/invalid-sdk/pubspec.lock @@ -1,4 +1,11 @@ packages: + args: + dependency: transitive + description: + name: args + url: "https://pub.dartlang.org" + source: hosted + version: "1.6.0" flutter: dependency: "direct main" description: flutter diff --git a/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock b/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock index ea3ed219f86..5ce5c348b4a 100644 --- a/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock +++ b/syft/pkg/cataloger/dart/test-fixtures/missing-sdk/pubspec.lock @@ -1,4 +1,11 @@ packages: + args: + dependency: transitive + description: + name: args + url: "https://pub.dartlang.org" + source: hosted + version: "1.6.0" flutter: dependency: "direct main" description: flutter From 44e689b0c5474376bb9248be4148b11b3812e9e5 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 22 Jan 2025 10:53:53 -0500 Subject: [PATCH 3/3] fix linting issues Signed-off-by: Alex Goodman --- syft/pkg/cataloger/dart/parse_pubspec_lock.go | 11 ++++------- syft/pkg/cataloger/dart/parse_pubspec_lock_test.go | 3 ++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/syft/pkg/cataloger/dart/parse_pubspec_lock.go b/syft/pkg/cataloger/dart/parse_pubspec_lock.go index 0970dd5126f..2d5a998d82c 100644 --- a/syft/pkg/cataloger/dart/parse_pubspec_lock.go +++ b/syft/pkg/cataloger/dart/parse_pubspec_lock.go @@ -7,9 +7,9 @@ import ( "regexp" "sort" + "github.com/Masterminds/semver" "gopkg.in/yaml.v3" - "github.com/Masterminds/semver" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" @@ -85,12 +85,9 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment if err != nil { log.Tracef("failed to resolve %s SDK version for package %s: %v", sdkName, name, err) continue - - } else { - log.Tracef("resolved %s SDK version for package %s to %s", sdkName, name, sdkVersion) - pkg.Version = sdkVersion - p.Packages[name] = pkg } + pkg.Version = sdkVersion + p.Packages[name] = pkg } names = append(names, name) @@ -168,7 +165,7 @@ func parseMinimumSdkVersion(constraint string) (string, error) { } // Read "version" subexpression (see 2. above) into version variable - version := []byte{} + var version []byte matchIndex := re.FindStringSubmatchIndex(constraint) version = re.ExpandString(version, "$version", constraint, matchIndex) diff --git a/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go b/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go index c002bf2c023..702e2cc3380 100644 --- a/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go +++ b/syft/pkg/cataloger/dart/parse_pubspec_lock_test.go @@ -3,11 +3,12 @@ package dart import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" - "github.com/stretchr/testify/assert" ) func TestParsePubspecLock(t *testing.T) {