diff --git a/extractor/filesystem/language/golang/gomod/extractor.go b/extractor/filesystem/language/golang/gomod/extractor.go new file mode 100644 index 0000000..7642f8b --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/extractor.go @@ -0,0 +1,153 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gomod extracts go.mod files. +package gomod + +import ( + "context" + "fmt" + "io" + "io/fs" + "path/filepath" + "strings" + + "golang.org/x/exp/maps" + "golang.org/x/mod/modfile" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" +) + +const goEcosystem string = "Go" + +// Extractor extracts go packages from a go.mod file, +// including the stdlib version by using the top level go version +// +// The output is not sorted and will not be in a consistent order +type Extractor struct{} + +// Name of the extractor. +func (e Extractor) Name() string { return "go/gomod" } + +// Version of the extractor. +func (e Extractor) Version() int { return 0 } + +// Requirements of the extractor. +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +// FileRequired returns true if the specified file matches go.mod files. +func (e Extractor) FileRequired(path string, fileInfo fs.FileInfo) bool { + return filepath.Base(path) == "go.mod" +} + +// Extract extracts packages from a go.mod file passed through the scan input. +func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + b, err := io.ReadAll(input.Reader) + if err != nil { + return nil, fmt.Errorf("could not read %s: %w", input.Path, err) + } + parsedLockfile, err := modfile.Parse(input.Path, b, nil) + if err != nil { + return nil, fmt.Errorf("could not extract from %s: %w", input.Path, err) + } + + // Store the packages in a map since they might be overwritten by later entries. + type mapKey struct { + name string + version string + } + packages := map[mapKey]*extractor.Inventory{} + + for _, require := range parsedLockfile.Require { + name := require.Mod.Path + version := strings.TrimPrefix(require.Mod.Version, "v") + packages[mapKey{name: name, version: version}] = &extractor.Inventory{ + Name: name, + Version: version, + Locations: []string{input.Path}, + } + } + + // Apply go.mod replace directives to the identified packages by updating their + // names+versions as instructed by the directive. + for _, replace := range parsedLockfile.Replace { + var replacements []mapKey + + if replace.Old.Version == "" { + // If the version to replace is omitted, all versions of the module are replaced. + for k, pkg := range packages { + if pkg.Name == replace.Old.Path { + replacements = append(replacements, k) + } + } + } else { + // If the version to replace is specified only that specific version of the + // module is replaced. + s := mapKey{name: replace.Old.Path, version: strings.TrimPrefix(replace.Old.Version, "v")} + + // A `replace` directive has no effect if the name or version to replace is not present. + if _, ok := packages[s]; ok { + replacements = []mapKey{s} + } + } + + for _, replacement := range replacements { + packages[replacement] = &extractor.Inventory{ + Name: replace.New.Path, + Version: strings.TrimPrefix(replace.New.Version, "v"), + Locations: []string{input.Path}, + } + } + } + + // Add the Go stdlib as an explicit dependency. + if parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" { + packages[mapKey{name: "stdlib"}] = &extractor.Inventory{ + Name: "stdlib", + Version: parsedLockfile.Go.Version, + Locations: []string{input.Path}, + } + } + + // The map values might have changed after replacement so we need to run another + // deduplication pass. + dedupedPs := map[mapKey]*extractor.Inventory{} + for _, p := range packages { + dedupedPs[mapKey{name: p.Name, version: p.Version}] = p + } + return maps.Values(dedupedPs), nil +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(i *extractor.Inventory) (*purl.PackageURL, error) { + return &purl.PackageURL{ + Type: purl.TypeGolang, + Name: i.Name, + Version: i.Version, + }, nil +} + +// ToCPEs is not applicable as this extractor does not infer CPEs from the Inventory. +func (e Extractor) ToCPEs(i *extractor.Inventory) ([]string, error) { return []string{}, nil } + +// Ecosystem returns the OSV Ecosystem of the software extracted by this extractor. +func (e Extractor) Ecosystem(i *extractor.Inventory) (string, error) { + return goEcosystem, nil +} + +var _ filesystem.Extractor = Extractor{} diff --git a/extractor/filesystem/language/golang/gomod/extractor_test.go b/extractor/filesystem/language/golang/gomod/extractor_test.go new file mode 100644 index 0000000..5bf53fa --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/extractor_test.go @@ -0,0 +1,285 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomod_test + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gomod" + "github.com/google/osv-scalibr/testing/extracttest" +) + +func TestExtractor_FileRequired(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputPath string + want bool + }{ + { + inputPath: "", + want: false, + }, + { + inputPath: "go.mod", + want: true, + }, + { + inputPath: "path/to/my/go.mod", + want: true, + }, + { + inputPath: "path/to/my/go.mod/file", + want: false, + }, + { + inputPath: "path/to/my/go.mod.file", + want: false, + }, + { + inputPath: "path.to.my.go.mod", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.inputPath, func(t *testing.T) { + t.Parallel() + e := gomod.Extractor{} + got := e.FileRequired(tt.inputPath, nil) + if got != tt.want { + t.Errorf("FileRequired(%s) got = %v, want %v", tt.inputPath, got, tt.want) + } + }) + } +} + +func TestExtractor_Extract(t *testing.T) { + t.Parallel() + + tests := []extracttest.TestTableEntry{ + { + Name: "invalid", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/not-go-mod.mod", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + }, + { + Name: "no packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/empty.mod", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "one package", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/one-package.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "github.com/BurntSushi/toml", + Version: "1.0.0", + Locations: []string{"testdata/one-package.mod"}, + }, + }, + }, + { + Name: "two packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/two-packages.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "github.com/BurntSushi/toml", + Version: "1.0.0", + Locations: []string{"testdata/two-packages.mod"}, + }, + { + Name: "gopkg.in/yaml.v2", + Version: "2.4.0", + Locations: []string{"testdata/two-packages.mod"}, + }, + { + Name: "stdlib", + Version: "1.17", + Locations: []string{"testdata/two-packages.mod"}, + }, + }, + }, + { + Name: "indirect packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/indirect-packages.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "github.com/BurntSushi/toml", + Version: "1.0.0", + Locations: []string{"testdata/indirect-packages.mod"}, + }, + { + Name: "gopkg.in/yaml.v2", + Version: "2.4.0", + Locations: []string{"testdata/indirect-packages.mod"}, + }, + { + Name: "github.com/mattn/go-colorable", + Version: "0.1.9", + Locations: []string{"testdata/indirect-packages.mod"}, + }, + { + Name: "github.com/mattn/go-isatty", + Version: "0.0.14", + Locations: []string{"testdata/indirect-packages.mod"}, + }, + { + Name: "golang.org/x/sys", + Version: "0.0.0-20210630005230-0f9fa26af87c", + Locations: []string{"testdata/indirect-packages.mod"}, + }, + { + Name: "stdlib", + Version: "1.17", + Locations: []string{"testdata/indirect-packages.mod"}, + }, + }, + }, + { + Name: "replacements_ one", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/replace-one.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "example.com/fork/net", + Version: "1.4.5", + Locations: []string{"testdata/replace-one.mod"}, + }, + }, + }, + { + Name: "replacements_ mixed", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/replace-mixed.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "example.com/fork/net", + Version: "1.4.5", + Locations: []string{"testdata/replace-mixed.mod"}, + }, + { + Name: "golang.org/x/net", + Version: "0.5.6", + Locations: []string{"testdata/replace-mixed.mod"}, + }, + }, + }, + { + Name: "replacements_ local", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/replace-local.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "./fork/net", + Version: "", + Locations: []string{"testdata/replace-local.mod"}, + }, + { + Name: "github.com/BurntSushi/toml", + Version: "1.0.0", + Locations: []string{"testdata/replace-local.mod"}, + }, + }, + }, + { + Name: "replacements_ different", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/replace-different.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "example.com/fork/foe", + Version: "1.4.5", + Locations: []string{"testdata/replace-different.mod"}, + }, + { + Name: "example.com/fork/foe", + Version: "1.4.2", + Locations: []string{"testdata/replace-different.mod"}, + }, + }, + }, + { + Name: "replacements_ not required", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/replace-not-required.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "golang.org/x/net", + Version: "0.5.6", + Locations: []string{"testdata/replace-not-required.mod"}, + }, + { + Name: "github.com/BurntSushi/toml", + Version: "1.0.0", + Locations: []string{"testdata/replace-not-required.mod"}, + }, + }, + }, + { + Name: "replacements_ no version", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/replace-no-version.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "example.com/fork/net", + Version: "1.4.5", + Locations: []string{"testdata/replace-no-version.mod"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + extr := gomod.Extractor{} + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } + }) + } +} diff --git a/extractor/filesystem/language/golang/gomod/testdata/empty.mod b/extractor/filesystem/language/golang/gomod/testdata/empty.mod new file mode 100644 index 0000000..e69de29 diff --git a/extractor/filesystem/language/golang/gomod/testdata/indirect-packages.mod b/extractor/filesystem/language/golang/gomod/testdata/indirect-packages.mod new file mode 100644 index 0000000..bda0e40 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/indirect-packages.mod @@ -0,0 +1,14 @@ +module my-library + +go 1.17 + +require ( + github.com/BurntSushi/toml v1.0.0 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/not-go-mod.mod b/extractor/filesystem/language/golang/gomod/testdata/not-go-mod.mod new file mode 100644 index 0000000..88baf50 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/not-go-mod.mod @@ -0,0 +1 @@ +this is not a go.mod file! diff --git a/extractor/filesystem/language/golang/gomod/testdata/one-package.mod b/extractor/filesystem/language/golang/gomod/testdata/one-package.mod new file mode 100644 index 0000000..6413971 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/one-package.mod @@ -0,0 +1,5 @@ +module my-library + +require ( + github.com/BurntSushi/toml v1.0.0 +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/replace-different.mod b/extractor/filesystem/language/golang/gomod/testdata/replace-different.mod new file mode 100644 index 0000000..49cc76b --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/replace-different.mod @@ -0,0 +1,9 @@ +require ( + golang.org/x/net v1.2.3 + golang.org/x/net v0.5.6 +) + +replace ( + golang.org/x/net v1.2.3 => example.com/fork/foe v1.4.5 + golang.org/x/net v0.5.6 => example.com/fork/foe v1.4.2 +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/replace-local.mod b/extractor/filesystem/language/golang/gomod/testdata/replace-local.mod new file mode 100644 index 0000000..e847de8 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/replace-local.mod @@ -0,0 +1,8 @@ +require ( + golang.org/x/net v1.2.3 + github.com/BurntSushi/toml v1.0.0 +) + +replace ( + golang.org/x/net v1.2.3 => ./fork/net +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/replace-mixed.mod b/extractor/filesystem/language/golang/gomod/testdata/replace-mixed.mod new file mode 100644 index 0000000..2f1a587 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/replace-mixed.mod @@ -0,0 +1,8 @@ +require ( + golang.org/x/net v1.2.3 + golang.org/x/net v0.5.6 +) + +replace ( + golang.org/x/net v1.2.3 => example.com/fork/net v1.4.5 +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/replace-no-version.mod b/extractor/filesystem/language/golang/gomod/testdata/replace-no-version.mod new file mode 100644 index 0000000..4802d6f --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/replace-no-version.mod @@ -0,0 +1,8 @@ +require ( + golang.org/x/net v1.2.3 + golang.org/x/net v0.5.6 +) + +replace ( + golang.org/x/net => example.com/fork/net v1.4.5 +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/replace-not-required.mod b/extractor/filesystem/language/golang/gomod/testdata/replace-not-required.mod new file mode 100644 index 0000000..c7c45ce --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/replace-not-required.mod @@ -0,0 +1,8 @@ +require ( + golang.org/x/net v0.5.6 + github.com/BurntSushi/toml v1.0.0 +) + +replace ( + golang.org/x/net v1.2.3 => example.com/fork/net v1.4.5 +) diff --git a/extractor/filesystem/language/golang/gomod/testdata/replace-one.mod b/extractor/filesystem/language/golang/gomod/testdata/replace-one.mod new file mode 100644 index 0000000..01c8169 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/replace-one.mod @@ -0,0 +1,5 @@ +require ( + golang.org/x/net v1.2.3 +) + +replace golang.org/x/net v1.2.3 => example.com/fork/net v1.4.5 diff --git a/extractor/filesystem/language/golang/gomod/testdata/two-packages.mod b/extractor/filesystem/language/golang/gomod/testdata/two-packages.mod new file mode 100644 index 0000000..1f3c6d9 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/two-packages.mod @@ -0,0 +1,8 @@ +module my-library + +go 1.17 + +require ( + github.com/BurntSushi/toml v1.0.0 + gopkg.in/yaml.v2 v2.4.0 +) diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index 7409499..0d2db25 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -30,6 +30,7 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/containers/containerd" "github.com/google/osv-scalibr/extractor/filesystem/language/dotnet/packageslockjson" "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gobinary" + "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gomod" javaarchive "github.com/google/osv-scalibr/extractor/filesystem/language/java/archive" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagejson" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" @@ -76,7 +77,10 @@ var ( poetrylock.Extractor{}, } // Go extractors. - Go []filesystem.Extractor = []filesystem.Extractor{gobinary.New(gobinary.DefaultConfig())} + Go []filesystem.Extractor = []filesystem.Extractor{ + gobinary.New(gobinary.DefaultConfig()), + &gomod.Extractor{}, + } // R extractors R []filesystem.Extractor = []filesystem.Extractor{renvlock.Extractor{}} // Ruby extractors. @@ -127,7 +131,6 @@ var ( Untested []filesystem.Extractor = []filesystem.Extractor{ osv.Wrapper{ExtractorName: "cpp/conan", ExtractorVersion: 0, PURLType: purl.TypeConan, Extractor: lockfile.ConanLockExtractor{}}, osv.Wrapper{ExtractorName: "dart/pubspec", ExtractorVersion: 0, PURLType: purl.TypePub, Extractor: lockfile.PubspecLockExtractor{}}, - osv.Wrapper{ExtractorName: "go/gomod", ExtractorVersion: 0, PURLType: purl.TypeGolang, Extractor: lockfile.GoLockExtractor{}}, osv.Wrapper{ExtractorName: "java/gradle", ExtractorVersion: 0, PURLType: purl.TypeMaven, Extractor: lockfile.GradleLockExtractor{}}, osv.Wrapper{ExtractorName: "java/pomxml", ExtractorVersion: 0, PURLType: purl.TypeMaven, Extractor: lockfile.MavenLockExtractor{}}, osv.Wrapper{ExtractorName: "javascript/yarn", ExtractorVersion: 0, PURLType: purl.TypeNPM, Extractor: lockfile.YarnLockExtractor{}},