diff --git a/core/cmd/scan.go b/core/cmd/scan.go index 2fc08d4..25641bf 100644 --- a/core/cmd/scan.go +++ b/core/cmd/scan.go @@ -16,9 +16,9 @@ import ( ) var ( - showDetails bool - url bool - silent bool + showDetailsFlag bool + urlFlag bool + silentFlag bool ) var ScanCommand = &cobra.Command{ @@ -26,8 +26,9 @@ var ScanCommand = &cobra.Command{ Short: "Scan a folder", Long: "Scan a folder", Run: func(cmd *cobra.Command, args []string) { - fmt.Println(config.Default.NameAndVersion()) - + if !silentFlag { + fmt.Println(config.Default.NameAndVersion()) + } if IsDatabaseUpdateAvailable() { UpdateDatabase() } @@ -36,48 +37,55 @@ var ScanCommand = &cobra.Command{ defer closeDb(db) for _, path := range args { - fmt.Printf("Scanning %s ...\n", path) - if err := common.ValidateDirectory(path); err != nil { - fmt.Printf("path %s not found\n", path) - return - } scanPath(path, db) } }, } func init() { - ScanCommand.Flags().BoolVarP(&showDetails, "details", "d", false, "show details") - ScanCommand.Flags().BoolVarP(&url, "url", "u", false, "url instead of path") - ScanCommand.Flags().BoolVarP(&silent, "silent", "s", false, "silent") + ScanCommand.Flags().BoolVarP(&showDetailsFlag, "details", "d", false, "show details") + ScanCommand.Flags().BoolVarP(&urlFlag, "url", "u", false, "url instead of path") + ScanCommand.Flags().BoolVarP(&silentFlag, "silent", "s", false, "silent") } func scanPath(path string, db *sql.DB) { - verbose := true - if silent { + var ( + err error + verbose = true + sourceType = es.PathSource + ) + + if silentFlag { verbose = false } - sourceType := es.PathSource - if url { + if urlFlag { sourceType = es.UrlSource } - source := es.NewSource(path, sourceType) - scanResults := scan.Inspect(source) - _printf := func(format string, a ...interface{}) { if verbose { fmt.Printf(format, a...) } } - _println := func(a ...interface{}) { if verbose { fmt.Println(a...) } } + source := es.NewSource(path, sourceType) + + err = es.ValidateSource(source) + if err != nil { + _println(err) + return + } + + _printf("Scanning %s ...\n", path) + + scanResults := scan.Inspect(source) + pkgVulQuerier := search.PackageVulnerabilityQuerier(db) for _, project := range scanResults.Projects { @@ -122,7 +130,7 @@ func scanPath(path string, db *sql.DB) { fmt.Printf("Version to update: %s\n", versionToUpdate) aliases := infoColor.Sprint(result.Vulnerabilities.AliasesSummary()) - if showDetails { + if showDetailsFlag { aliases = "" } @@ -137,7 +145,7 @@ func scanPath(path string, db *sql.DB) { printedDep[result.Query.ToString()] = true - if showDetails { + if showDetailsFlag { for _, vul := range result.Vulnerabilities { alias := vul.VulnerabilityId if len(vul.AliasesParsed()) > 0 { diff --git a/core/common/files.go b/core/common/files.go index 8e68ef6..42d1853 100644 --- a/core/common/files.go +++ b/core/common/files.go @@ -86,7 +86,7 @@ func ReadAllFile(filePath string) ([]byte, error) { return data, nil } -func DetectLineEnding(filePath string) string { +func DetectFileLineEnding(filePath string) string { content, err := ReadAllFile(filePath) if err != nil { return "\n" diff --git a/core/common/strings.go b/core/common/strings.go index 5673fc3..fdb728d 100644 --- a/core/common/strings.go +++ b/core/common/strings.go @@ -16,3 +16,11 @@ func Plural(count int, singular, plural string) string { } return singular } + +func DetectStringLineEnding(content string) string { + if strings.Contains(content, "\r\n") { + return "\r\n" + } + + return "\n" +} diff --git a/core/common/url.go b/core/common/url.go index cdae925..44c6cb7 100644 --- a/core/common/url.go +++ b/core/common/url.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net/http" + "time" ) func DownloadUrlContent(url string) ([]byte, error) { @@ -29,3 +30,29 @@ func DownloadUrlContent(url string) ([]byte, error) { return content, nil } + +func UrlExists(url string, timeLimit int) bool { + if timeLimit <= 0 { + timeLimit = 5 + } + client := http.Client{ + Timeout: time.Duration(timeLimit) * time.Second, + } + response, err := client.Head(url) + if err != nil { + return false + } + if response.StatusCode != http.StatusOK { + return false + } + return true +} + +func FirstUrlExists(urls []string, timeLimit int) string { + for _, url := range urls { + if UrlExists(url, timeLimit) { + return url + } + } + return "" +} diff --git a/core/ecosystem/cratesio/cargo_lock.go b/core/ecosystem/cratesio/cargo_lock.go index c886884..cc0b119 100644 --- a/core/ecosystem/cratesio/cargo_lock.go +++ b/core/ecosystem/cratesio/cargo_lock.go @@ -2,31 +2,20 @@ package cratesio import ( "cvepack/core/common" - "cvepack/core/ecosystem" - "log" - "path/filepath" + es "cvepack/core/ecosystem" "strings" ) -func NewProjectFromCargoLock(path string) ecosystem.Project { - pkgs := ecosystem.Packages{} - file := filepath.Join(path, CargoLockFile) - cargoLockContent, err := common.ReadAllFile(file) - - if err != nil { - log.Println(err) - return ecosystem.NewProject(path, EcosystemName, pkgs) - } - - lines := strings.Split(string(cargoLockContent), common.DetectLineEnding(file)) +func parseCargoLockContent(content string) es.Packages { + pkgs := es.Packages{} + lines := strings.Split(content, common.DetectStringLineEnding(content)) for i, line := range lines { if line == "[[package]]" { name := strings.Replace(lines[i+1][7:], "\"", "", -1) version := strings.Replace(lines[i+2][9:], "\"", "", -1) - pkgs = append(pkgs, ecosystem.NewDefaultPackage(name, strings.TrimSpace(version), "")) + pkgs = append(pkgs, es.NewDefaultPackage(name, strings.TrimSpace(version), "")) } } - - return ecosystem.NewProject(path, EcosystemName, pkgs) + return pkgs } diff --git a/core/ecosystem/cratesio/cargo_lock_test.go b/core/ecosystem/cratesio/cargo_lock_test.go deleted file mode 100644 index 9e56bcf..0000000 --- a/core/ecosystem/cratesio/cargo_lock_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package cratesio - -import ( - "testing" -) - -func Test_BuildProjectFromCargoLock(t *testing.T) { - project := NewProjectFromCargoLock("./testdata") - - if len(project.Packages()) != 180 { - t.Errorf("Expected project to have 180 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/cratesio/const.go b/core/ecosystem/cratesio/const.go index bb38a93..dbe77fb 100644 --- a/core/ecosystem/cratesio/const.go +++ b/core/ecosystem/cratesio/const.go @@ -6,3 +6,7 @@ const ( CargoFile = "Cargo.toml" CargoLockFile = "Cargo.lock" ) + +func EcosystemTitle() string { + return EcosystemLanguage + " (" + EcosystemName + ")" +} diff --git a/core/ecosystem/cratesio/detection.go b/core/ecosystem/cratesio/detection.go deleted file mode 100644 index ef33ac0..0000000 --- a/core/ecosystem/cratesio/detection.go +++ /dev/null @@ -1,14 +0,0 @@ -package cratesio - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectCargoToml(path string) bool { - return common.FileExists(filepath.Join(path, CargoFile)) -} - -func DetectCargoLock(path string) bool { - return common.FileExists(filepath.Join(path, CargoLockFile)) -} diff --git a/core/ecosystem/cratesio/project_builder.go b/core/ecosystem/cratesio/project_builder.go deleted file mode 100644 index b128b1b..0000000 --- a/core/ecosystem/cratesio/project_builder.go +++ /dev/null @@ -1,14 +0,0 @@ -package cratesio - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - - if DetectCargoToml(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromCargoLock, - "Cargo.lock detected!") - } - - return nil -} diff --git a/core/ecosystem/cratesio/provider.go b/core/ecosystem/cratesio/provider.go new file mode 100644 index 0000000..a5492a1 --- /dev/null +++ b/core/ecosystem/cratesio/provider.go @@ -0,0 +1,30 @@ +package cratesio + +import ( + es "cvepack/core/ecosystem" + "errors" + "fmt" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + cargoFilePath := provider.GetFirstExistingPath(CargoFile) + + // check for Cargo.toml first + if cargoFilePath != "" { + cargoLockFilePath := provider.GetFirstExistingPath(CargoLockFile) + + // then check for Cargo.lock + if cargoLockFilePath != "" { + content, err := es.ProviderPathContent(provider, cargoLockFilePath) + if err != nil { + return nil, err + } + return es.NewProject(provider.Source().Value, EcosystemName, parseCargoLockContent(content)), nil + } + + // otherwise, fall back to Cargo.toml + // todo: support Cargo.toml + } + + return nil, errors.New(fmt.Sprintf("no %s project found", EcosystemTitle())) +} diff --git a/core/ecosystem/cratesio/provider_test.go b/core/ecosystem/cratesio/provider_test.go new file mode 100644 index 0000000..11bbd44 --- /dev/null +++ b/core/ecosystem/cratesio/provider_test.go @@ -0,0 +1,19 @@ +package cratesio + +import ( + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProviderFromPath("./testdata") + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 180 { + t.Errorf("Expected project to have 180 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/golang/detection.go b/core/ecosystem/golang/detection.go deleted file mode 100644 index 71ecd61..0000000 --- a/core/ecosystem/golang/detection.go +++ /dev/null @@ -1,15 +0,0 @@ -package golang - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectGoMod(path string) bool { - return common.FileExists(filepath.Join(path, "go.mod")) - -} - -func DetectGoSum(path string) bool { - return common.FileExists(filepath.Join(path, "go.sum")) -} diff --git a/core/ecosystem/golang/gosum.go b/core/ecosystem/golang/gosum.go index 39b96f1..e746f9b 100644 --- a/core/ecosystem/golang/gosum.go +++ b/core/ecosystem/golang/gosum.go @@ -2,22 +2,13 @@ package golang import ( "cvepack/core/common" - "cvepack/core/ecosystem" - "log" - "path/filepath" + es "cvepack/core/ecosystem" "strings" ) -func NewProjectFromGoSum(path string) ecosystem.Project { - pkgs := ecosystem.Packages{} - file := filepath.Join(path, "go.sum") - goSumContent, err := common.ReadAllFile(file) - if err != nil { - log.Println(err) - return ecosystem.NewProject(path, EcosystemName, pkgs) - } - - lines := strings.Split(string(goSumContent), common.DetectLineEnding(file)) +func parsePackagesFromGoSumContent(content string) es.Packages { + pkgs := es.Packages{} + lines := strings.Split(content, common.DetectStringLineEnding(content)) for _, line := range lines { if line == "" { @@ -31,8 +22,7 @@ func NewProjectFromGoSum(path string) ecosystem.Project { name := parts[0] version := parts[1][1:] - pkgs.Append(ecosystem.NewDefaultPackage(name, version, "")) + pkgs.Append(es.NewDefaultPackage(name, version, "")) } - - return ecosystem.NewProject(path, EcosystemName, pkgs) + return pkgs } diff --git a/core/ecosystem/golang/gosum_test.go b/core/ecosystem/golang/gosum_test.go deleted file mode 100644 index 84ff5c8..0000000 --- a/core/ecosystem/golang/gosum_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package golang - -import "testing" - -func Test_NewProjectFromGoSum(t *testing.T) { - project := NewProjectFromGoSum("./testdata") - - if len(project.Packages()) != 35 { - t.Errorf("Expected project to have 35 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/golang/project.go b/core/ecosystem/golang/project.go new file mode 100644 index 0000000..152361c --- /dev/null +++ b/core/ecosystem/golang/project.go @@ -0,0 +1,7 @@ +package golang + +import "cvepack/core/ecosystem" + +func NewProject(source ecosystem.Source, packages []ecosystem.Package) ecosystem.Project { + return ecosystem.NewProject(source.Value, EcosystemName, packages) +} diff --git a/core/ecosystem/golang/project_builder.go b/core/ecosystem/golang/project_builder.go deleted file mode 100644 index 68bb616..0000000 --- a/core/ecosystem/golang/project_builder.go +++ /dev/null @@ -1,14 +0,0 @@ -package golang - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - - if DetectGoMod(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromGoSum, - "go.sum detected!") - } - - return nil -} diff --git a/core/ecosystem/golang/provider.go b/core/ecosystem/golang/provider.go new file mode 100644 index 0000000..5287156 --- /dev/null +++ b/core/ecosystem/golang/provider.go @@ -0,0 +1,24 @@ +package golang + +import ( + es "cvepack/core/ecosystem" + "errors" + "fmt" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + goSumPath := provider.GetFirstExistingPath(GoSum) + + if goSumPath != "" { + content, err := es.ProviderPathContent(provider, goSumPath) + if err != nil { + return nil, err + } + return es.NewProject( + provider.Source().Value, + EcosystemName, + parsePackagesFromGoSumContent(content)), nil + } + + return nil, errors.New(fmt.Sprintf("no %s found", GoSum)) +} diff --git a/core/ecosystem/golang/provider_test.go b/core/ecosystem/golang/provider_test.go new file mode 100644 index 0000000..33b01af --- /dev/null +++ b/core/ecosystem/golang/provider_test.go @@ -0,0 +1,19 @@ +package golang + +import ( + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProviderFromPath("./testdata") + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 35 { + t.Errorf("Expected project to have 35 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/maven/const.go b/core/ecosystem/maven/const.go index 1e5ea51..aafef4d 100644 --- a/core/ecosystem/maven/const.go +++ b/core/ecosystem/maven/const.go @@ -5,3 +5,7 @@ const ( EcosystemName = "Maven" PomXml = "pom.xml" ) + +func EcosystemTitle() string { + return EcosystemLanguage + " (" + EcosystemName + ")" +} diff --git a/core/ecosystem/maven/dectection.go b/core/ecosystem/maven/dectection.go deleted file mode 100644 index 1313c3d..0000000 --- a/core/ecosystem/maven/dectection.go +++ /dev/null @@ -1,10 +0,0 @@ -package maven - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectPomXml(path string) bool { - return common.FileExists(filepath.Join(path, PomXml)) -} diff --git a/core/ecosystem/maven/pomxml.go b/core/ecosystem/maven/pomxml.go index 7d06a31..d4428cb 100644 --- a/core/ecosystem/maven/pomxml.go +++ b/core/ecosystem/maven/pomxml.go @@ -3,10 +3,7 @@ package maven import ( "cvepack/core/ecosystem" "encoding/xml" - "io" "log" - "os" - "path/filepath" ) type pomXml struct { @@ -31,35 +28,10 @@ type pomXml struct { } `xml:"dependencies"` } -func NewProjectFromPomXml(path string) ecosystem.Project { - file := filepath.Join(path, PomXml) - return ecosystem.NewProject(path, EcosystemName, scanPackagesFromPomXml(file)) -} - -func scanPackagesFromPomXml(file string) ecosystem.Packages { +func parsePackagesFromPomXmlContent(content string) ecosystem.Packages { pkgs := ecosystem.Packages{} - - xmlFile, err := os.Open(file) - if err != nil { - log.Println(err) - return pkgs - } - - defer func(xmlFile *os.File) { - err := xmlFile.Close() - if err != nil { - panic(err) - } - }(xmlFile) - - byteValue, err := io.ReadAll(xmlFile) - if err != nil { - log.Println(err) - return pkgs - } - var project pomXml - err = xml.Unmarshal(byteValue, &project) + err := xml.Unmarshal([]byte(content), &project) if err != nil { log.Println(err) return pkgs diff --git a/core/ecosystem/maven/pomxml_test.go b/core/ecosystem/maven/pomxml_test.go deleted file mode 100644 index d3e4388..0000000 --- a/core/ecosystem/maven/pomxml_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package maven - -import ( - "testing" -) - -func Test_NewProjectFromPomXml(t *testing.T) { - project := NewProjectFromPomXml("./testdata") - - if len(project.Packages()) != 2 { - t.Errorf("Expected project to have 2 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/maven/project_builder.go b/core/ecosystem/maven/project_builder.go deleted file mode 100644 index 90965fc..0000000 --- a/core/ecosystem/maven/project_builder.go +++ /dev/null @@ -1,14 +0,0 @@ -package maven - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - - if DetectPomXml(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromPomXml, - "pom.xml detected!") - } - - return nil -} diff --git a/core/ecosystem/maven/provider.go b/core/ecosystem/maven/provider.go new file mode 100644 index 0000000..3c96122 --- /dev/null +++ b/core/ecosystem/maven/provider.go @@ -0,0 +1,21 @@ +package maven + +import ( + es "cvepack/core/ecosystem" + "errors" + "fmt" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + pomXmlPath := provider.GetFirstExistingPath(PomXml) + + if pomXmlPath != "" { + content, err := es.ProviderPathContent(provider, pomXmlPath) + if err != nil { + return nil, err + } + return es.NewProject(provider.Source().Value, EcosystemName, parsePackagesFromPomXmlContent(content)), nil + } + + return nil, errors.New(fmt.Sprintf("no %s project found", EcosystemTitle())) +} diff --git a/core/ecosystem/maven/provider_test.go b/core/ecosystem/maven/provider_test.go new file mode 100644 index 0000000..afc7ab8 --- /dev/null +++ b/core/ecosystem/maven/provider_test.go @@ -0,0 +1,19 @@ +package maven + +import ( + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProviderFromPath("./testdata") + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 2 { + t.Errorf("Expected project to have 2 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/npm/detection.go b/core/ecosystem/npm/detection.go deleted file mode 100644 index 066ad39..0000000 --- a/core/ecosystem/npm/detection.go +++ /dev/null @@ -1,18 +0,0 @@ -package npm - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectPackageJson(path string) bool { - return common.FileExists(filepath.Join(path, PackageFile)) -} - -func DetectPackageLockJson(path string) bool { - return common.FileExists(filepath.Join(path, PackageLockFile)) -} - -func DetectNodeModules(path string) bool { - return common.DirectoryExists(filepath.Join(path, NodeModulesFolder)) -} diff --git a/core/ecosystem/npm/file.go b/core/ecosystem/npm/file.go index 4e7b805..ba06557 100644 --- a/core/ecosystem/npm/file.go +++ b/core/ecosystem/npm/file.go @@ -5,19 +5,19 @@ import ( "encoding/json" ) -func fileToPackageLockJson(filePath string) (*packageLockJson, error) { - content, err := common.ReadAllFile(filePath) - if err != nil { - return nil, err - } - - var pkgLock packageLockJson - if err := json.Unmarshal(content, &pkgLock); err != nil { - return nil, err - } - - return &pkgLock, nil -} +//func fileToPackageLockJson(filePath string) (*packageLockJson, error) { +// content, err := common.ReadAllFile(filePath) +// if err != nil { +// return nil, err +// } +// +// var pkgLock packageLockJson +// if err := json.Unmarshal(content, &pkgLock); err != nil { +// return nil, err +// } +// +// return &pkgLock, nil +//} func fileToPackageJson(filePath string) (*packageJson, error) { content, err := common.ReadAllFile(filePath) diff --git a/core/ecosystem/npm/node-modules.go b/core/ecosystem/npm/node-modules.go index 0d00649..7129259 100644 --- a/core/ecosystem/npm/node-modules.go +++ b/core/ecosystem/npm/node-modules.go @@ -9,11 +9,6 @@ import ( "slices" ) -func NewProjectFromNodeModules(path string) ecosystem.Project { - path = filepath.Join(path, NodeModulesFolder) - return ecosystem.NewProject(path, EcosystemName, scanNodeModules(path)) -} - func scanNodeModules(path string) ecosystem.Packages { var packages ecosystem.Packages var excludedPaths = []string{".bin", ".pnpm"} diff --git a/core/ecosystem/npm/package_json.go b/core/ecosystem/npm/package_json.go new file mode 100644 index 0000000..021af55 --- /dev/null +++ b/core/ecosystem/npm/package_json.go @@ -0,0 +1,6 @@ +package npm + +type packageJson struct { + Name string `json:"name"` + Version string `json:"version"` +} diff --git a/core/ecosystem/npm/package_lock_json.go b/core/ecosystem/npm/package_lock_json.go new file mode 100644 index 0000000..0ed6e32 --- /dev/null +++ b/core/ecosystem/npm/package_lock_json.go @@ -0,0 +1,26 @@ +package npm + +import "encoding/json" + +type packageLockJson struct { + Name string `json:"name"` + Version string `json:"version"` + LockfileVersion int `json:"lockfileVersion"` + Requires bool `json:"requires"` + Packages map[string]packageLockPackage `json:"packages"` +} + +type packageLockPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Dependencies map[string]string `json:"dependencies"` +} + +func stringToPackageLockJson(content string) (*packageLockJson, error) { + var pkgLock packageLockJson + if err := json.Unmarshal([]byte(content), &pkgLock); err != nil { + return nil, err + } + + return &pkgLock, nil +} diff --git a/core/ecosystem/npm/packagejson.go b/core/ecosystem/npm/packagejson.go deleted file mode 100644 index 0f7c527..0000000 --- a/core/ecosystem/npm/packagejson.go +++ /dev/null @@ -1,58 +0,0 @@ -package npm - -import ( - "cvepack/core/ecosystem" - "log" - "path/filepath" -) - -type packageJson struct { - Name string `json:"name"` - Version string `json:"version"` -} - -type packageLockJson struct { - Name string `json:"name"` - Version string `json:"version"` - LockfileVersion int `json:"lockfileVersion"` - Requires bool `json:"requires"` - Packages map[string]packageLockPackage `json:"packages"` -} - -type packageLockPackage struct { - Name string `json:"name"` - Version string `json:"version"` - Dependencies map[string]string `json:"dependencies"` -} - -func NewProjectFromPackageLockJson(path string) ecosystem.Project { - file := filepath.Join(path, PackageLockFile) - pkgLock, err := fileToPackageLockJson(file) - - // todo: return error - if err != nil { - log.Printf("Error while loadding file %s : %s", file, err) - return ecosystem.NewProject(path, EcosystemName, ecosystem.Packages{}) - } - - return PackageLockJsonToProject(pkgLock, path) -} - -// PackageLockJsonToProject -// transform packageLockJson to Project -func PackageLockJsonToProject(pkgLock *packageLockJson, source string) ecosystem.Project { - pkgs := ecosystem.Packages{} - - for pkgKey, pkg := range pkgLock.Packages { - if pkgKey == "" { - // reference to the package itself or an installed node_modules package, skip - continue - } else if pkg.Name != "" && pkg.Name != pkgKey { - // reference to a local package - pkgKey = pkg.Name - } - pkgs = append(pkgs, NewPackage(pkgKey, pkg.Version)) - } - - return ecosystem.NewProject(source, EcosystemName, pkgs) -} diff --git a/core/ecosystem/npm/packagejson_test.go b/core/ecosystem/npm/packagejson_test.go deleted file mode 100644 index fd5c008..0000000 --- a/core/ecosystem/npm/packagejson_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package npm - -import "testing" - -func Test_BuildFromPackageLockJson(t *testing.T) { - npmProject := NewProjectFromPackageLockJson("./testdata") - if npmProject.Ecosystem() != "npm" { - t.Errorf("Expected project name to be 'npm', got %s", npmProject.Ecosystem()) - } - - if len(npmProject.Packages()) != 409 { - t.Errorf("Expected project to have 409 packages, got %d", len(npmProject.Packages())) - } -} diff --git a/core/ecosystem/npm/project.go b/core/ecosystem/npm/project.go new file mode 100644 index 0000000..7ba3f64 --- /dev/null +++ b/core/ecosystem/npm/project.go @@ -0,0 +1,33 @@ +package npm + +import ( + "cvepack/core/ecosystem" + "log" +) + +func PackageLockJsonToProject(pkgLock *packageLockJson, source string) ecosystem.Project { + pkgs := ecosystem.Packages{} + + for pkgKey, pkg := range pkgLock.Packages { + if pkgKey == "" { + // reference to the package itself or an installed node_modules package, skip + continue + } else if pkg.Name != "" && pkg.Name != pkgKey { + // reference to a local package + pkgKey = pkg.Name + } + pkgs = append(pkgs, NewPackage(pkgKey, pkg.Version)) + } + + return ecosystem.NewProject(source, EcosystemName, pkgs) +} + +func StringPackageLockJsonToProject(pkgJson string, source string) ecosystem.Project { + pkg, err := stringToPackageLockJson(pkgJson) + if err != nil { + log.Printf("Error while loadding file %s : %s", pkgJson, err) + return ecosystem.NewProject(source, EcosystemName, ecosystem.Packages{}) + } + + return PackageLockJsonToProject(pkg, source) +} diff --git a/core/ecosystem/npm/project_builder.go b/core/ecosystem/npm/project_builder.go deleted file mode 100644 index ca1fb68..0000000 --- a/core/ecosystem/npm/project_builder.go +++ /dev/null @@ -1,19 +0,0 @@ -package npm - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - if DetectPackageJson(path) { - if DetectPackageLockJson(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromPackageLockJson, - "package-lock.json detected!") - } else if DetectNodeModules(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromNodeModules, - "node_modules detected!") - } - } - - return nil -} diff --git a/core/ecosystem/npm/provider.go b/core/ecosystem/npm/provider.go new file mode 100644 index 0000000..8b96ef7 --- /dev/null +++ b/core/ecosystem/npm/provider.go @@ -0,0 +1,38 @@ +package npm + +import ( + es "cvepack/core/ecosystem" + "errors" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + packageJsonPath := provider.GetFirstExistingPath(PackageFile) + + // check for package.json first + if packageJsonPath != "" { + packageJsonLockPath := provider.GetFirstExistingPath(PackageLockFile) + + // then check for package-lock.json + if packageJsonLockPath != "" { + content, err := es.ProviderPathContent(provider, packageJsonLockPath) + if err != nil { + return nil, err + } + return StringPackageLockJsonToProject(content, packageJsonLockPath), nil + } + + // otherwise, check for node_modules + nodeModulesFolder := provider.GetFirstExistingPath(NodeModulesFolder) + if nodeModulesFolder != "" { + return es.NewProject(nodeModulesFolder, EcosystemName, scanNodeModules(nodeModulesFolder)), nil + } + + // otherwise, check for yarn.lock + // TODO support yarn.lock + + // otherwise, use package.json + // TODO support package.json + } + + return nil, errors.New("no package.json found") +} diff --git a/core/ecosystem/npm/provider_test.go b/core/ecosystem/npm/provider_test.go new file mode 100644 index 0000000..17e38bc --- /dev/null +++ b/core/ecosystem/npm/provider_test.go @@ -0,0 +1,19 @@ +package npm + +import ( + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProviderFromPath("./testdata") + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 409 { + t.Errorf("Expected project to have 409 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/nuget/const.go b/core/ecosystem/nuget/const.go index 850ee7b..ab4a634 100644 --- a/core/ecosystem/nuget/const.go +++ b/core/ecosystem/nuget/const.go @@ -4,3 +4,7 @@ const ( EcosystemLanguage = ".Net" EcosystemName = "NuGet" ) + +func EcosystemTitle() string { + return EcosystemLanguage + " (" + EcosystemName + ")" +} diff --git a/core/ecosystem/nuget/detection.go b/core/ecosystem/nuget/detection.go deleted file mode 100644 index d4efe47..0000000 --- a/core/ecosystem/nuget/detection.go +++ /dev/null @@ -1,21 +0,0 @@ -package nuget - -import ( - "os" - "strings" -) - -func DetectSln(path string) string { - files, err := os.ReadDir(path) - if err != nil { - return "" - } - - for _, file := range files { - if !file.IsDir() && strings.HasSuffix(file.Name(), ".sln") { - return file.Name() - } - } - - return "" -} diff --git a/core/ecosystem/nuget/project_builder.go b/core/ecosystem/nuget/project_builder.go deleted file mode 100644 index 2f93f8e..0000000 --- a/core/ecosystem/nuget/project_builder.go +++ /dev/null @@ -1,15 +0,0 @@ -package nuget - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - slnFile := DetectSln(path) - if slnFile != "" { - return ecosystem.NewProjectBuilder( - func(path string) ecosystem.Project { - return NewProjectFromSln(path, slnFile) - }, - ".sln file detected!") - } - return nil -} diff --git a/core/ecosystem/nuget/provider.go b/core/ecosystem/nuget/provider.go new file mode 100644 index 0000000..8f1abae --- /dev/null +++ b/core/ecosystem/nuget/provider.go @@ -0,0 +1,53 @@ +package nuget + +import ( + es "cvepack/core/ecosystem" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + if provider.Source().Type() == es.UrlSource { + return nil, errors.New(fmt.Sprintf("NuGet url scanning not supported yet")) + } else if provider.Source().Type() == es.UnknownSource { + return nil, errors.New(fmt.Sprintf("Unknown source")) + } + + files, err := os.ReadDir(provider.Source().Value) + if err != nil { + return nil, err + } + + var slnFile string + + for _, file := range files { + if !file.IsDir() && strings.HasSuffix(file.Name(), ".sln") { + slnFile = file.Name() + break + } + } + + if slnFile != "" { + log.Printf("Found solution file: %s", slnFile) + pkgs := es.Packages{} + file := filepath.Join(provider.Source().Value, slnFile) + csprojFiles := scanCsprojFromSln(file) + + if len(csprojFiles) == 0 { + return nil, errors.New("no csproj file(s) found") + } + + for _, csproj := range csprojFiles { + pkgs.Append(scanPackagesFromCsProjXml( + filepath.Join(provider.Source().Value, csproj))...) + } + + return es.NewProject(provider.Source().Value, EcosystemName, pkgs), nil + } + return nil, errors.New("no .sln file found") + +} diff --git a/core/ecosystem/nuget/provider_test.go b/core/ecosystem/nuget/provider_test.go new file mode 100644 index 0000000..06fd8da --- /dev/null +++ b/core/ecosystem/nuget/provider_test.go @@ -0,0 +1,20 @@ +package nuget + +import ( + es "cvepack/core/ecosystem" + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProvider(es.NewPathSource("./testdata")) + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 6 { + t.Errorf("Expected project to have 6 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/nuget/sln.go b/core/ecosystem/nuget/sln.go index 98fd10a..5cfa686 100644 --- a/core/ecosystem/nuget/sln.go +++ b/core/ecosystem/nuget/sln.go @@ -7,23 +7,13 @@ import ( "io" "log" "os" - "path/filepath" "regexp" "strings" ) -var slnCsprojRegex = regexp.MustCompile(`"([^"]+\.csproj)"`) - -func NewProjectFromSln(path string, slnFile string) ecosystem.Project { - file := filepath.Join(path, slnFile) - - pkgs := ecosystem.Packages{} - for _, csproj := range scanCsprojFromSln(file) { - pkgs.Append(scanPackagesFromCsProjXml(filepath.Join(path, csproj))...) - } - - return ecosystem.NewProject(path, EcosystemName, pkgs) -} +var ( + slnCsprojRegex = regexp.MustCompile(`"([^"]+\.csproj)"`) +) func scanCsprojFromSln(file string) []string { content, err := common.ReadAllFile(file) @@ -32,7 +22,7 @@ func scanCsprojFromSln(file string) []string { return []string{} } - lineEnding := common.DetectLineEnding(file) + lineEnding := common.DetectFileLineEnding(file) lines := strings.Split(string(content), lineEnding) var csprojPaths []string diff --git a/core/ecosystem/nuget/sln_test.go b/core/ecosystem/nuget/sln_test.go deleted file mode 100644 index 8ae8426..0000000 --- a/core/ecosystem/nuget/sln_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package nuget - -import ( - "testing" -) - -func Test_NewProjectFromSln(t *testing.T) { - project := NewProjectFromSln("./testdata", "project.sln") - - if len(project.Packages()) != 6 { - t.Errorf("Expected project to have 6 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/packagist/composer_lock.go b/core/ecosystem/packagist/composer_lock.go index 5b6cd3c..8edb3e9 100644 --- a/core/ecosystem/packagist/composer_lock.go +++ b/core/ecosystem/packagist/composer_lock.go @@ -1,9 +1,7 @@ package packagist import ( - "cvepack/core/ecosystem" - "log" - "path/filepath" + es "cvepack/core/ecosystem" ) type composerLockFile struct { @@ -16,21 +14,13 @@ type composerLockPackage struct { Version string `json:"version"` } -func NewProjectFromComposerLock(path string) ecosystem.Project { - pkgs := ecosystem.Packages{} - file := filepath.Join(path, ComposerLockFile) - composerLock, err := readComposerLockFile(file) - - if err == nil { - for _, pkg := range composerLock.Packages { - pkgs = append(pkgs, ecosystem.NewDefaultPackage(pkg.Name, pkg.Version, "")) - } - for _, pkg := range composerLock.PackagesDev { - pkgs = append(pkgs, ecosystem.NewDefaultPackage(pkg.Name, pkg.Version, "")) - } - } else { - log.Printf("Error while loadding file %s : %s", file, err) +func parsePackagesFromComposerLockFile(composerLock composerLockFile) es.Packages { + pkgs := es.Packages{} + for _, pkg := range composerLock.Packages { + pkgs = append(pkgs, es.NewDefaultPackage(pkg.Name, pkg.Version, "")) } - - return ecosystem.NewProject(path, EcosystemName, pkgs) + for _, pkg := range composerLock.PackagesDev { + pkgs = append(pkgs, es.NewDefaultPackage(pkg.Name, pkg.Version, "")) + } + return pkgs } diff --git a/core/ecosystem/packagist/composer_lock_test.go b/core/ecosystem/packagist/composer_lock_test.go deleted file mode 100644 index e840835..0000000 --- a/core/ecosystem/packagist/composer_lock_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package packagist - -import ( - "testing" -) - -func Test_BuildProjectFromComposerLock(t *testing.T) { - project := NewProjectFromComposerLock("./testdata") - if project.Ecosystem() != "Packagist" { - t.Errorf("Expected project name to be 'Packagist', got %s", project.Ecosystem()) - } - - if len(project.Packages()) != 43 { - t.Errorf("Expected project to have 43 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/packagist/detection.go b/core/ecosystem/packagist/detection.go deleted file mode 100644 index 425ea2d..0000000 --- a/core/ecosystem/packagist/detection.go +++ /dev/null @@ -1,14 +0,0 @@ -package packagist - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectComposerJson(path string) bool { - return common.FileExists(filepath.Join(path, ComposerFile)) -} - -func DetectComposerLock(path string) bool { - return common.FileExists(filepath.Join(path, ComposerLockFile)) -} diff --git a/core/ecosystem/packagist/project_builder.go b/core/ecosystem/packagist/project_builder.go deleted file mode 100644 index d7257be..0000000 --- a/core/ecosystem/packagist/project_builder.go +++ /dev/null @@ -1,14 +0,0 @@ -package packagist - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - - if DetectComposerJson(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromComposerLock, - "composer.json detected!") - } - - return nil -} diff --git a/core/ecosystem/packagist/provider.go b/core/ecosystem/packagist/provider.go new file mode 100644 index 0000000..c632471 --- /dev/null +++ b/core/ecosystem/packagist/provider.go @@ -0,0 +1,42 @@ +package packagist + +import ( + es "cvepack/core/ecosystem" + "encoding/json" + "errors" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + composerPath := provider.GetFirstExistingPath(ComposerFile) + + if composerPath != "" { + composerLockPath := provider.GetFirstExistingPath(ComposerLockFile) + + if composerLockPath != "" { + content, err := es.ProviderPathContent(provider, composerLockPath) + if err != nil { + return nil, err + } + + composerLock, err := stringToComposerLockJson(content) + if err != nil { + return nil, err + } + + pkgs := parsePackagesFromComposerLockFile(*composerLock) + return es.NewProject(composerLockPath, EcosystemName, pkgs), nil + } + + return nil, errors.New("no composer.lock found") + } + + return nil, errors.New("no composer.json found") +} + +func stringToComposerLockJson(content string) (*composerLockFile, error) { + var composerLockFile composerLockFile + if err := json.Unmarshal([]byte(content), &composerLockFile); err != nil { + return nil, err + } + return &composerLockFile, nil +} diff --git a/core/ecosystem/packagist/provider_test.go b/core/ecosystem/packagist/provider_test.go new file mode 100644 index 0000000..001ef3d --- /dev/null +++ b/core/ecosystem/packagist/provider_test.go @@ -0,0 +1,20 @@ +package packagist + +import ( + es "cvepack/core/ecosystem" + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProvider(es.NewPathSource("./testdata")) + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 43 { + t.Errorf("Expected project to have 43 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/provider.go b/core/ecosystem/provider.go new file mode 100644 index 0000000..d2c5a26 --- /dev/null +++ b/core/ecosystem/provider.go @@ -0,0 +1,27 @@ +package ecosystem + +import "cvepack/core/common" + +type Provider interface { + // GetPaths returns a list of paths to the file in the provider's source + GetPaths(file string) []string + + // GetFirstExistingPath retrieve the path of the first existing file in the provider's source + GetFirstExistingPath(file string) string + + // Source returns the provider's source + Source() Source +} + +func ProviderPathContent(provider Provider, file string) (string, error) { + switch provider.Source().Type() { + case PathSource: + content, err := common.ReadAllFile(file) + return string(content), err + case UrlSource: + content, err := common.DownloadUrlContent(file) + return string(content), err + default: + return "", ErrorUnknownSourceType(provider.Source()) + } +} diff --git a/core/ecosystem/pypi/detection.go b/core/ecosystem/pypi/detection.go deleted file mode 100644 index 7399647..0000000 --- a/core/ecosystem/pypi/detection.go +++ /dev/null @@ -1,18 +0,0 @@ -package pypi - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectPyProjectToml(path string) bool { - return common.FileExists(filepath.Join(path, PyProjectToml)) -} - -func DetectPoetryLock(path string) bool { - return common.FileExists(filepath.Join(path, PoetryLock)) -} - -func DetectPdmLock(path string) bool { - return common.FileExists(filepath.Join(path, PdmLock)) -} diff --git a/core/ecosystem/pypi/lock-parser.go b/core/ecosystem/pypi/lock_parser.go similarity index 53% rename from core/ecosystem/pypi/lock-parser.go rename to core/ecosystem/pypi/lock_parser.go index 6bec209..e5d0b0d 100644 --- a/core/ecosystem/pypi/lock-parser.go +++ b/core/ecosystem/pypi/lock_parser.go @@ -15,7 +15,7 @@ func parseLockContent(file string) (ecosystem.Packages, error) { return pkgs, err } - lineEnding := common.DetectLineEnding(file) + lineEnding := common.DetectFileLineEnding(file) lines := strings.Split(string(lockContent), lineEnding) for i, line := range lines { @@ -28,3 +28,20 @@ func parseLockContent(file string) (ecosystem.Packages, error) { return pkgs, nil } + +func parsePackagesLockContent(content string) ecosystem.Packages { + lineEnding := common.DetectStringLineEnding(content) + lines := strings.Split(content, lineEnding) + pkgs := ecosystem.Packages{} + + for i, line := range lines { + if line == "[[package]]" { + name := strings.Replace(lines[i+1][7:], "\"", "", -1) + version := strings.Replace(lines[i+2][9:], "\"", "", -1) + pkgs = append(pkgs, ecosystem.NewDefaultPackage(name, strings.TrimSpace(version), "")) + } + } + + return pkgs + +} diff --git a/core/ecosystem/pypi/lock_parser_test.go b/core/ecosystem/pypi/lock_parser_test.go new file mode 100644 index 0000000..805c081 --- /dev/null +++ b/core/ecosystem/pypi/lock_parser_test.go @@ -0,0 +1,27 @@ +package pypi + +import ( + "testing" +) + +func Test_parseLockContent_with_poetryLock(t *testing.T) { + file := "./testdata/poetry.lock" + pkgs, err := parseLockContent(file) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if len(pkgs) != 74 { + t.Errorf("Expected 74 packages, got %d", len(pkgs)) + } +} + +func Test_parseLockContent_with_pdmLock(t *testing.T) { + file := "./testdata/pdm.lock" + pkgs, err := parseLockContent(file) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if len(pkgs) != 47 { + t.Errorf("Expected 47 packages, got %d", len(pkgs)) + } +} diff --git a/core/ecosystem/pypi/pdm_lock.go b/core/ecosystem/pypi/pdm_lock.go deleted file mode 100644 index a18e7dd..0000000 --- a/core/ecosystem/pypi/pdm_lock.go +++ /dev/null @@ -1,19 +0,0 @@ -package pypi - -import ( - "cvepack/core/ecosystem" - "log" - "path/filepath" -) - -func NewProjectFromPdmLock(path string) ecosystem.Project { - file := filepath.Join(path, PdmLock) - - pkgs, err := parseLockContent(file) - - if err != nil { - log.Println(err) - } - - return ecosystem.NewProject(path, EcosystemName, pkgs) -} diff --git a/core/ecosystem/pypi/pdm_lock_test.go b/core/ecosystem/pypi/pdm_lock_test.go deleted file mode 100644 index bd1224f..0000000 --- a/core/ecosystem/pypi/pdm_lock_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package pypi - -import ( - "testing" -) - -func Test_BuildProjectFromPdmLock(t *testing.T) { - project := NewProjectFromPdmLock("./testdata") - - if len(project.Packages()) != 47 { - t.Errorf("Expected project to have 47 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/pypi/poetry_lock.go b/core/ecosystem/pypi/poetry_lock.go deleted file mode 100644 index c7e498e..0000000 --- a/core/ecosystem/pypi/poetry_lock.go +++ /dev/null @@ -1,18 +0,0 @@ -package pypi - -import ( - "cvepack/core/ecosystem" - "log" - "path/filepath" -) - -func NewProjectFromPoetryLock(path string) ecosystem.Project { - file := filepath.Join(path, PoetryLock) - pkgs, err := parseLockContent(file) - - if err != nil { - log.Println(err) - } - - return ecosystem.NewProject(path, EcosystemName, pkgs) -} diff --git a/core/ecosystem/pypi/poetry_lock_test.go b/core/ecosystem/pypi/poetry_lock_test.go deleted file mode 100644 index 245b0cc..0000000 --- a/core/ecosystem/pypi/poetry_lock_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package pypi - -import ( - "testing" -) - -func Test_BuildProjectFromPoetryLock(t *testing.T) { - project := NewProjectFromPoetryLock("./testdata") - - if len(project.Packages()) != 74 { - t.Errorf("Expected project to have 74 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/pypi/project_builder.go b/core/ecosystem/pypi/project_builder.go deleted file mode 100644 index 7736cac..0000000 --- a/core/ecosystem/pypi/project_builder.go +++ /dev/null @@ -1,20 +0,0 @@ -package pypi - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - - if DetectPyProjectToml(path) { - if DetectPoetryLock(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromPoetryLock, - "poetry.lock detected!") - } else if DetectPdmLock(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromPdmLock, - "pdm.lock detected!") - } - } - - return nil -} diff --git a/core/ecosystem/pypi/provider.go b/core/ecosystem/pypi/provider.go new file mode 100644 index 0000000..a46c7e9 --- /dev/null +++ b/core/ecosystem/pypi/provider.go @@ -0,0 +1,33 @@ +package pypi + +import ( + es "cvepack/core/ecosystem" + "errors" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + pyProjectTomlPath := provider.GetFirstExistingPath(PyProjectToml) + + if pyProjectTomlPath != "" { + poetryLockPath := provider.GetFirstExistingPath(PoetryLock) + + if poetryLockPath != "" { + content, err := es.ProviderPathContent(provider, poetryLockPath) + if err != nil { + return nil, err + } + return es.NewProject(provider.Source().Value, EcosystemName, parsePackagesLockContent(content)), nil + } + + pdmLockPath := provider.GetFirstExistingPath(PdmLock) + if pdmLockPath != "" { + content, err := es.ProviderPathContent(provider, pdmLockPath) + if err != nil { + return nil, err + } + return es.NewProject(provider.Source().Value, EcosystemName, parsePackagesLockContent(content)), nil + } + } + + return nil, errors.New("no pypi project found") +} diff --git a/core/ecosystem/pypi/provider_test.go b/core/ecosystem/pypi/provider_test.go new file mode 100644 index 0000000..5b62d14 --- /dev/null +++ b/core/ecosystem/pypi/provider_test.go @@ -0,0 +1,20 @@ +package pypi + +import ( + es "cvepack/core/ecosystem" + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProvider(es.NewSource("./testdata", es.PathSource)) + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 74 { + t.Errorf("Expected project to have 74 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/rubygems/dectection.go b/core/ecosystem/rubygems/dectection.go deleted file mode 100644 index d6e6ea1..0000000 --- a/core/ecosystem/rubygems/dectection.go +++ /dev/null @@ -1,14 +0,0 @@ -package rubygems - -import ( - "cvepack/core/common" - "path/filepath" -) - -func DetectGemFile(path string) bool { - return common.FileExists(filepath.Join(path, GemFile)) -} - -func DetectGemFileLock(path string) bool { - return common.FileExists(filepath.Join(path, GemFileLock)) -} diff --git a/core/ecosystem/rubygems/gemfile_lock.go b/core/ecosystem/rubygems/gemfile_lock.go index a13500f..db8b3da 100644 --- a/core/ecosystem/rubygems/gemfile_lock.go +++ b/core/ecosystem/rubygems/gemfile_lock.go @@ -4,8 +4,6 @@ import ( "cvepack/core/common" "cvepack/core/ecosystem" "fmt" - "log" - "path/filepath" "regexp" "strings" ) @@ -17,16 +15,22 @@ type gemFilePackage struct { Version string } -func NewProjectFromGemFileLock(path string) ecosystem.Project { - pkgs := ecosystem.Packages{} - file := filepath.Join(path, GemFileLock) - gemLockContent, err := common.ReadAllFile(file) - if err != nil { - log.Println(err) - return ecosystem.NewProject(path, EcosystemName, pkgs) +func parsePackageString(text string) (gemFilePackage, error) { + match := packageRegex.FindStringSubmatch(text) + + if match != nil { + return gemFilePackage{ + Name: match[1], + Version: match[2], + }, nil } + return gemFilePackage{}, fmt.Errorf("no match found") +} - lines := strings.Split(string(gemLockContent), common.DetectLineEnding(file)) +func parsePackagesGemFileLockContent(content string) ecosystem.Packages { + pkgs := ecosystem.Packages{} + lineEnding := common.DetectStringLineEnding(content) + lines := strings.Split(content, lineEnding) specsSection := false for _, line := range lines { @@ -50,17 +54,5 @@ func NewProjectFromGemFileLock(path string) ecosystem.Project { } } - return ecosystem.NewProject(path, EcosystemName, pkgs) -} - -func parsePackageString(text string) (gemFilePackage, error) { - match := packageRegex.FindStringSubmatch(text) - - if match != nil { - return gemFilePackage{ - Name: match[1], - Version: match[2], - }, nil - } - return gemFilePackage{}, fmt.Errorf("no match found") + return pkgs } diff --git a/core/ecosystem/rubygems/gemfile_lock_test.go b/core/ecosystem/rubygems/gemfile_lock_test.go deleted file mode 100644 index 37415fb..0000000 --- a/core/ecosystem/rubygems/gemfile_lock_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package rubygems - -import ( - "testing" -) - -func Test_BuildProjectFromGemFileLock(t *testing.T) { - project := NewProjectFromGemFileLock("./testdata") - - if len(project.Packages()) != 46 { - t.Errorf("Expected project to have 46 packages, got %d", len(project.Packages())) - } -} diff --git a/core/ecosystem/rubygems/project_builder.go b/core/ecosystem/rubygems/project_builder.go deleted file mode 100644 index bbd6a34..0000000 --- a/core/ecosystem/rubygems/project_builder.go +++ /dev/null @@ -1,14 +0,0 @@ -package rubygems - -import "cvepack/core/ecosystem" - -func ProjectBuilder(path string) *ecosystem.ProjectBuilder { - - if DetectGemFileLock(path) { - return ecosystem.NewProjectBuilder( - NewProjectFromGemFileLock, - "Gemfile.lock detected!") - } - - return nil -} diff --git a/core/ecosystem/rubygems/provider.go b/core/ecosystem/rubygems/provider.go new file mode 100644 index 0000000..6737b51 --- /dev/null +++ b/core/ecosystem/rubygems/provider.go @@ -0,0 +1,19 @@ +package rubygems + +import ( + es "cvepack/core/ecosystem" + "errors" +) + +func NewProjectFromProvider(provider es.Provider) (es.Project, error) { + gemFileLockPath := provider.GetFirstExistingPath(GemFileLock) + + if gemFileLockPath != "" { + content, err := es.ProviderPathContent(provider, gemFileLockPath) + if err != nil { + return nil, err + } + return es.NewProject(provider.Source().Value, EcosystemName, parsePackagesGemFileLockContent(content)), nil + } + return nil, errors.New("no rubygems project found") +} diff --git a/core/ecosystem/rubygems/provider_test.go b/core/ecosystem/rubygems/provider_test.go new file mode 100644 index 0000000..3bb3383 --- /dev/null +++ b/core/ecosystem/rubygems/provider_test.go @@ -0,0 +1,20 @@ +package rubygems + +import ( + es "cvepack/core/ecosystem" + "cvepack/core/scan/files" + "testing" +) + +func Test_NewProjectFromProvider(t *testing.T) { + provider := files.NewProvider(es.NewSource("./testdata", es.PathSource)) + project, err := NewProjectFromProvider(provider) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(project.Packages()) != 46 { + t.Errorf("Expected project to have 46 packages, got %d", len(project.Packages())) + } +} diff --git a/core/ecosystem/source.go b/core/ecosystem/source.go index 93377aa..f4f8950 100644 --- a/core/ecosystem/source.go +++ b/core/ecosystem/source.go @@ -1,5 +1,12 @@ package ecosystem +import ( + "cvepack/core/common" + "errors" + "fmt" + "strings" +) + type SourceType string const ( @@ -17,7 +24,15 @@ func NewSource(value string, sourceType SourceType) Source { return Source{Value: value, _type: sourceType} } -func (s *Source) Type() SourceType { +func NewPathSource(value string) Source { + return NewSource(value, PathSource) +} + +func NewUrlSource(value string) Source { + return NewSource(value, UrlSource) +} + +func (s Source) Type() SourceType { return s._type } @@ -31,3 +46,23 @@ func StringToSourceType(t string) SourceType { return UnknownSource } } + +func ValidateSource(source Source) error { + if strings.TrimSpace(source.Value) == "" { + return errors.New("source is empty") + } + if source.Type() == UnknownSource { + return errors.New(fmt.Sprintf("unknown source: %s", source.Value)) + } + + if source.Type() == PathSource { + if err := common.ValidateDirectory(source.Value); err != nil { + return err + } + } + return nil +} + +func ErrorUnknownSourceType(source Source) error { + return errors.New(fmt.Sprintf("unknown source for %s", source.Value)) +} diff --git a/core/scan/files/provider.go b/core/scan/files/provider.go new file mode 100644 index 0000000..62dbe5b --- /dev/null +++ b/core/scan/files/provider.go @@ -0,0 +1,38 @@ +package files + +import ( + "cvepack/core/common" + "cvepack/core/ecosystem" + "path/filepath" +) + +type Provider struct { + source ecosystem.Source +} + +func NewProvider(source ecosystem.Source) *Provider { + return &Provider{source} +} + +func NewProviderFromPath(path string) *Provider { + return NewProvider(ecosystem.NewPathSource(path)) +} + +func (p *Provider) GetPaths(file string) []string { + fp := filepath.Join(p.source.Value, file) + return []string{fp} +} + +func (p *Provider) Source() ecosystem.Source { + return p.source +} + +func (p *Provider) GetFirstExistingPath(file string) string { + paths := p.GetPaths(file) + for _, path := range paths { + if common.FileExists(path) { + return path + } + } + return "" +} diff --git a/core/scan/github/detection.go b/core/scan/github/detection.go new file mode 100644 index 0000000..41915c3 --- /dev/null +++ b/core/scan/github/detection.go @@ -0,0 +1,7 @@ +package github + +import "cvepack/core/ecosystem" + +func DetectGithubRepoUrl(source ecosystem.Source) bool { + return NewUrl(source).IsValid +} diff --git a/core/scan/github/github.go b/core/scan/github/github.go new file mode 100644 index 0000000..419d017 --- /dev/null +++ b/core/scan/github/github.go @@ -0,0 +1,52 @@ +package github + +import ( + "cvepack/core/common" + "cvepack/core/ecosystem" + "regexp" +) + +var ( + urlTpl = "https://raw.githubusercontent.com/{repo}/{branch}/{filepath}" +) + +type Url struct { + Source ecosystem.Source + Repo string + IsValid bool +} + +func NewUrl(source ecosystem.Source) *Url { + gh := &Url{Source: source} + validateUrl(gh) + return gh +} + +func (gh *Url) GetFileRawUrl(branch, filepath string) string { + return common.ReplacePlaceholders(urlTpl, map[string]string{ + "repo": gh.Repo, + "branch": branch, + "filepath": filepath, + }) +} + +func (gh *Url) GeneratePath(filepath string) string { + branch := "master" + return common.ReplacePlaceholders("{repo}/{branch}/{filepath}", map[string]string{ + "repo": gh.Repo, + "branch": branch, + "filepath": filepath, + }) +} + +func validateUrl(gh *Url) { + regexStr := `^(?:https?:\/\/)?github\.com\/([a-zA-Z0-9\-_]+\/[a-zA-Z0-9\-_]+)` + regex := regexp.MustCompile(regexStr) + matches := regex.FindStringSubmatch(gh.Source.Value) + if len(matches) == 0 { + return + } + + gh.Repo = matches[1] + gh.IsValid = true +} diff --git a/core/scan/github/github_test.go b/core/scan/github/github_test.go new file mode 100644 index 0000000..45997f4 --- /dev/null +++ b/core/scan/github/github_test.go @@ -0,0 +1,57 @@ +package github + +import ( + "cvepack/core/ecosystem" + "testing" +) + +func Test_DetectGithubRepoUrl(t *testing.T) { + type scenario struct { + url string + isValid bool + repo string + } + + tests := []scenario{ + {"https://github.com/someuser", false, ""}, + {"https://github.com/someuser/repo", true, "someuser/repo"}, + {"github.com/some_user/repo", true, "some_user/repo"}, + {"github.com/someUser1/repo/", true, "someUser1/repo"}, + {"http://github.com/someUser1/repo/", true, "someUser1/repo"}, + {"https://github.com/someUser_1/repo_1/blob/master/README.md", true, "someUser_1/repo_1"}, + {"https://github.com/someuser/repo/issues", true, "someuser/repo"}, + } + + for _, test := range tests { + source := ecosystem.NewSource(test.url, ecosystem.UrlSource) + gh := NewUrl(source) + if gh.IsValid != test.isValid { + t.Errorf("Expected %s to be %t", test.url, test.isValid) + } + if gh.Repo != test.repo { + t.Errorf("Expected repo to be %s, got %s", test.repo, gh.Repo) + } + } +} + +func Test_GetFileRawUrl(t *testing.T) { + type scenario struct { + url string + branch string + filepath string + expected string + } + + tests := []scenario{ + {"https://github.com/someuser/repo", "main", "README.md", "https://raw.githubusercontent.com/someuser/repo/main/README.md"}, + } + + for _, test := range tests { + source := ecosystem.NewSource(test.url, ecosystem.UrlSource) + gh := NewUrl(source) + got := gh.GetFileRawUrl(test.branch, test.filepath) + if got != test.expected { + t.Errorf("Expected %s, got %s", test.expected, got) + } + } +} diff --git a/core/scan/github/provider.go b/core/scan/github/provider.go new file mode 100644 index 0000000..5d3285a --- /dev/null +++ b/core/scan/github/provider.go @@ -0,0 +1,48 @@ +package github + +import ( + "cvepack/core/common" + "cvepack/core/ecosystem" +) + +var ( + branches = []string{"master", "main"} + defaultBranch = "master" +) + +type Provider struct { + source ecosystem.Source + url *Url +} + +func NewProvider(source ecosystem.Source) *Provider { + return &Provider{source, NewUrl(source)} +} + +func (p *Provider) GetPaths(file string) []string { + paths := make([]string, 0) + for _, branch := range branches { + paths = append(paths, p.url.GetFileRawUrl(branch, file)) + } + return paths +} + +func (p *Provider) Source() ecosystem.Source { + return p.source +} + +func (p *Provider) GetFirstExistingPath(file string) string { + paths := p.GetPaths(file) + for _, path := range paths { + if urlExists := common.UrlExists(path, 8); urlExists { + return path + } + } + return "" +} + +// GetPathContent returns the content of path relative to the provider source +func (p *Provider) GetPathContent(file string) (string, error) { + content, err := common.DownloadUrlContent(file) + return string(content), err +} diff --git a/core/scan/github/scan.go b/core/scan/github/scan.go deleted file mode 100644 index d0096bd..0000000 --- a/core/scan/github/scan.go +++ /dev/null @@ -1,42 +0,0 @@ -package github - -import ( - "io" - "net/http" -) - -func getUrlContent(url string) (string, error) { - response, err := http.Get(url) - if err != nil { - return "", err - } - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - panic(err) - } - }(response.Body) - - body, err := io.ReadAll(response.Body) - if err != nil { - return "", err - } - - // Print the HTML content as a string - return string(body), nil -} - -//func ScanUrl(url string) []ecosystem.Project { -// content, err := getUrlContent(url) -// if err != nil { -// return []ecosystem.Project{} -// } -// -// -// if strings.Contains(content, fmt.Sprintf("title=\"%s\"", golang.GoMod)) { -// -// } -// if (golang.GoMod) -// -// -//} diff --git a/core/scan/provider_scanner.go b/core/scan/provider_scanner.go new file mode 100644 index 0000000..efe8274 --- /dev/null +++ b/core/scan/provider_scanner.go @@ -0,0 +1,59 @@ +package scan + +import ( + es "cvepack/core/ecosystem" + "cvepack/core/ecosystem/cratesio" + "cvepack/core/ecosystem/golang" + "cvepack/core/ecosystem/maven" + "cvepack/core/ecosystem/npm" + "cvepack/core/ecosystem/nuget" + "cvepack/core/ecosystem/packagist" + "cvepack/core/ecosystem/pypi" + "cvepack/core/ecosystem/rubygems" +) + +func ProviderScanner(provider es.Provider) []es.Project { + projects := make([]es.Project, 0) + + cratesioProject, err := cratesio.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, cratesioProject) + } + + golangProject, err := golang.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, golangProject) + } + + mavenProject, err := maven.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, mavenProject) + } + + npmProject, err := npm.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, npmProject) + } + + nugetProject, err := nuget.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, nugetProject) + } + + packagistProject, err := packagist.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, packagistProject) + } + + pypiProject, err := pypi.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, pypiProject) + } + + rubyGems, err := rubygems.NewProjectFromProvider(provider) + if err == nil { + projects = append(projects, rubyGems) + } + + return projects +} diff --git a/core/scan/scan_url.go b/core/scan/scan_url.go deleted file mode 100644 index c9156d0..0000000 --- a/core/scan/scan_url.go +++ /dev/null @@ -1,26 +0,0 @@ -package scan - -//type ScanUrl struct { -// Url string -// Projects []ecosystem.Project -// Verbose bool -//} -// -//func NewScanUrl(url string, verbose bool) *ScanUrl { -// return &ScanUrl{Url: url, Verbose: verbose} -//} -// -//func (scan *ScanUrl) Log(msg string) { -// if scan.Verbose { -// fmt.Println(msg) -// } -//} -// -//func (scan *ScanUrl) Run() []ecosystem.Project { -// projects := make([]ecosystem.Project, 0) -// if github.DetectGithubRepoUrl(scan.Url) { -// scan.Log("Detected GitHub repo") -// return projects -// } -// return projects -//} diff --git a/core/scan/scanner.go b/core/scan/scanner.go index e75af61..abd4dc4 100644 --- a/core/scan/scanner.go +++ b/core/scan/scanner.go @@ -2,17 +2,9 @@ package scan import ( "cvepack/core/ecosystem" - "cvepack/core/ecosystem/cratesio" - "cvepack/core/ecosystem/golang" - "cvepack/core/ecosystem/maven" - "cvepack/core/ecosystem/npm" - "cvepack/core/ecosystem/nuget" - "cvepack/core/ecosystem/packagist" - "cvepack/core/ecosystem/pypi" - "cvepack/core/ecosystem/rubygems" + "cvepack/core/scan/files" "cvepack/core/scan/github" "fmt" - "sync" ) func Inspect(source ecosystem.Source) *ScannerResult { @@ -27,88 +19,30 @@ func newScanner() *scanner { return &scanner{} } +// Scan scans a source for projects +// support url and path sources func (s *scanner) Scan(source ecosystem.Source) *ScannerResult { scanResult := NewScannerResult(source) defer scanResult.End() - switch source.Type() { + switch source.Type() { case ecosystem.UrlSource: scanResult.Projects = s.scanUrl(source) case ecosystem.PathSource: - scanResult.Projects = s.scanPath(source) + fileProvider := files.NewProvider(source) + scanResult.Projects = ProviderScanner(fileProvider) } return scanResult } -// scanPath scans a path for projects -// this is a builders based approach -// each builder is responsible for detecting and building a ecosystem project builder -func (s *scanner) scanPath(source ecosystem.Source) []ecosystem.Project { - path := source.Value - waitGroup := sync.WaitGroup{} - projects := make([]ecosystem.Project, 0) - builders := make([]ecosystem.ProjectBuilder, 0) - - npmProjectBuilder := npm.ProjectBuilder(path) - if npmProjectBuilder != nil { - builders = append(builders, *npmProjectBuilder) - } - - golangProjectBuilder := golang.ProjectBuilder(path) - if golangProjectBuilder != nil { - builders = append(builders, *golangProjectBuilder) - } - - packagistProjectBuilder := packagist.ProjectBuilder(path) - if packagistProjectBuilder != nil { - builders = append(builders, *packagistProjectBuilder) - } - - cratesioBuilder := cratesio.ProjectBuilder(path) - if cratesioBuilder != nil { - builders = append(builders, *cratesioBuilder) - } - - rubygemsBuilder := rubygems.ProjectBuilder(path) - if rubygemsBuilder != nil { - builders = append(builders, *rubygemsBuilder) - } - - pypiBuilder := pypi.ProjectBuilder(path) - if pypiBuilder != nil { - builders = append(builders, *pypiBuilder) - } - - nugetBuilder := nuget.ProjectBuilder(path) - if nugetBuilder != nil { - builders = append(builders, *nugetBuilder) - } - - mavenBuilder := maven.ProjectBuilder(path) - if mavenBuilder != nil { - builders = append(builders, *mavenBuilder) - } - - for _, builder := range builders { - waitGroup.Add(1) - b := builder - go func() { - projects = append(projects, b.Build(path)) - waitGroup.Done() - }() - } - - waitGroup.Wait() - return projects -} - +// scanUrl scan url for projects func (s *scanner) scanUrl(source ecosystem.Source) []ecosystem.Project { - fmt.Printf("Scanning url %s", source.Value) + fmt.Printf("Url %s\n", source.Value) projects := make([]ecosystem.Project, 0) - if github.DetectGithubRepoUrl(source.Value) { - fmt.Printf("Detected GitHub repo") - return projects + if github.DetectGithubRepoUrl(source) { + provider := github.NewProvider(source) + return ProviderScanner(provider) } return projects }