refactor scanner to support provider/url

core/cmd/scan.go

@@ -16,18 +16,19 @@ import (
 )
 
 var (
-	showDetails bool
-	url         bool
-	silent      bool
+	showDetailsFlag bool
+	urlFlag         bool
+	silentFlag      bool
 )
 
 var ScanCommand = &cobra.Command{
 	Use:   "scan",
 	Short: "Scan a folder",
 	Long:  "Scan a folder",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println(config.Default.NameAndVersion())
-
+		if !silentFlag {
+			fmt.Println(config.Default.NameAndVersion())
+		}
 		if IsDatabaseUpdateAvailable() {
 			UpdateDatabase()
 		}
@@ -36,48 +37,55 @@ var ScanCommand = &cobra.Command{
 		defer closeDb(db)
 
 		for _, path := range args {
-			fmt.Printf("Scanning %s ...\n", path)
-			if err := common.ValidateDirectory(path); err != nil {
-				fmt.Printf("path %s not found\n", path)
-				return
-			}
 			scanPath(path, db)
 		}
 	},
 }
 
 func init() {
-	ScanCommand.Flags().BoolVarP(&showDetails, "details", "d", false, "show details")
-	ScanCommand.Flags().BoolVarP(&url, "url", "u", false, "url instead of path")
-	ScanCommand.Flags().BoolVarP(&silent, "silent", "s", false, "silent")
+	ScanCommand.Flags().BoolVarP(&showDetailsFlag, "details", "d", false, "show details")
+	ScanCommand.Flags().BoolVarP(&urlFlag, "url", "u", false, "url instead of path")
+	ScanCommand.Flags().BoolVarP(&silentFlag, "silent", "s", false, "silent")
 }
 
 func scanPath(path string, db *sql.DB) {
-	verbose := true
-	if silent {
+	var (
+		err        error
+		verbose    = true
+		sourceType = es.PathSource
+	)
+
+	if silentFlag {
 		verbose = false
 	}
 
-	sourceType := es.PathSource
-	if url {
+	if urlFlag {
 		sourceType = es.UrlSource
 	}
 
-	source := es.NewSource(path, sourceType)
-	scanResults := scan.Inspect(source)
-
 	_printf := func(format string, a ...interface{}) {
 		if verbose {
 			fmt.Printf(format, a...)
 		}
 	}
-
 	_println := func(a ...interface{}) {
 		if verbose {
 			fmt.Println(a...)
 		}
 	}
 
+	source := es.NewSource(path, sourceType)
+
+	err = es.ValidateSource(source)
+	if err != nil {
+		_println(err)
+		return
+	}
+
+	_printf("Scanning %s ...\n", path)
+
+	scanResults := scan.Inspect(source)
+
 	pkgVulQuerier := search.PackageVulnerabilityQuerier(db)
 
 	for _, project := range scanResults.Projects {
@@ -122,7 +130,7 @@ func scanPath(path string, db *sql.DB) {
 				fmt.Printf("Version to update: %s\n", versionToUpdate)
 
 				aliases := infoColor.Sprint(result.Vulnerabilities.AliasesSummary())
-				if showDetails {
+				if showDetailsFlag {
 					aliases = ""
 				}
 
@@ -137,7 +145,7 @@ func scanPath(path string, db *sql.DB) {
 
 				printedDep[result.Query.ToString()] = true
 
-				if showDetails {
+				if showDetailsFlag {
 					for _, vul := range result.Vulnerabilities {
 						alias := vul.VulnerabilityId
 						if len(vul.AliasesParsed()) > 0 {

core/common/files.go

@@ -86,7 +86,7 @@ func ReadAllFile(filePath string) ([]byte, error) {
 	return data, nil
 }
 
-func DetectLineEnding(filePath string) string {
+func DetectFileLineEnding(filePath string) string {
 	content, err := ReadAllFile(filePath)
 	if err != nil {
 		return "\n"

core/common/strings.go

@@ -16,3 +16,11 @@ func Plural(count int, singular, plural string) string {
 	}
 	return singular
 }
+
+func DetectStringLineEnding(content string) string {
+	if strings.Contains(content, "\r\n") {
+		return "\r\n"
+	}
+
+	return "\n"
+}

core/common/url.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 )
 
 func DownloadUrlContent(url string) ([]byte, error) {
@@ -29,3 +30,29 @@ func DownloadUrlContent(url string) ([]byte, error) {
 
 	return content, nil
 }
+
+func UrlExists(url string, timeLimit int) bool {
+	if timeLimit <= 0 {
+		timeLimit = 5
+	}
+	client := http.Client{
+		Timeout: time.Duration(timeLimit) * time.Second,
+	}
+	response, err := client.Head(url)
+	if err != nil {
+		return false
+	}
+	if response.StatusCode != http.StatusOK {
+		return false
+	}
+	return true
+}
+
+func FirstUrlExists(urls []string, timeLimit int) string {
+	for _, url := range urls {
+		if UrlExists(url, timeLimit) {
+			return url
+		}
+	}
+	return ""
+}

core/ecosystem/cratesio/cargo_lock.go

@@ -2,31 +2,20 @@ package cratesio
 
 import (
 	"cvepack/core/common"
-	"cvepack/core/ecosystem"
-	"log"
-	"path/filepath"
+	es "cvepack/core/ecosystem"
 	"strings"
 )
 
-func NewProjectFromCargoLock(path string) ecosystem.Project {
-	pkgs := ecosystem.Packages{}
-	file := filepath.Join(path, CargoLockFile)
-	cargoLockContent, err := common.ReadAllFile(file)
-
-	if err != nil {
-		log.Println(err)
-		return ecosystem.NewProject(path, EcosystemName, pkgs)
-	}
-
-	lines := strings.Split(string(cargoLockContent), common.DetectLineEnding(file))
+func parseCargoLockContent(content string) es.Packages {
+	pkgs := es.Packages{}
+	lines := strings.Split(content, common.DetectStringLineEnding(content))
 
 	for i, line := range lines {
 		if line == "[[package]]" {
 			name := strings.Replace(lines[i+1][7:], "\"", "", -1)
 			version := strings.Replace(lines[i+2][9:], "\"", "", -1)
-			pkgs = append(pkgs, ecosystem.NewDefaultPackage(name, strings.TrimSpace(version), ""))
+			pkgs = append(pkgs, es.NewDefaultPackage(name, strings.TrimSpace(version), ""))
 		}
 	}
-
-	return ecosystem.NewProject(path, EcosystemName, pkgs)
+	return pkgs
 }

core/ecosystem/cratesio/const.go

@@ -6,3 +6,7 @@ const (
 	CargoFile         = "Cargo.toml"
 	CargoLockFile     = "Cargo.lock"
 )
+
+func EcosystemTitle() string {
+	return EcosystemLanguage + " (" + EcosystemName + ")"
+}

core/ecosystem/cratesio/provider.go

@@ -0,0 +1,30 @@
+package cratesio
+
+import (
+	es "cvepack/core/ecosystem"
+	"errors"
+	"fmt"
+)
+
+func NewProjectFromProvider(provider es.Provider) (es.Project, error) {
+	cargoFilePath := provider.GetFirstExistingPath(CargoFile)
+
+	// check for Cargo.toml first
+	if cargoFilePath != "" {
+		cargoLockFilePath := provider.GetFirstExistingPath(CargoLockFile)
+
+		// then check for Cargo.lock
+		if cargoLockFilePath != "" {
+			content, err := es.ProviderPathContent(provider, cargoLockFilePath)
+			if err != nil {
+				return nil, err
+			}
+			return es.NewProject(provider.Source().Value, EcosystemName, parseCargoLockContent(content)), nil
+		}
+
+		// otherwise, fall back to Cargo.toml
+		// todo: support Cargo.toml
+	}
+
+	return nil, errors.New(fmt.Sprintf("no %s project found", EcosystemTitle()))
+}

core/ecosystem/cratesio/provider_test.go

@@ -0,0 +1,19 @@
+package cratesio
+
+import (
+	"cvepack/core/scan/files"
+	"testing"
+)
+
+func Test_NewProjectFromProvider(t *testing.T) {
+	provider := files.NewProviderFromPath("./testdata")
+	project, err := NewProjectFromProvider(provider)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	if len(project.Packages()) != 180 {
+		t.Errorf("Expected project to have 180 packages, got %d", len(project.Packages()))
+	}
+}

core/ecosystem/golang/gosum.go

@@ -2,22 +2,13 @@ package golang
 
 import (
 	"cvepack/core/common"
-	"cvepack/core/ecosystem"
-	"log"
-	"path/filepath"
+	es "cvepack/core/ecosystem"
 	"strings"
 )
 
-func NewProjectFromGoSum(path string) ecosystem.Project {
-	pkgs := ecosystem.Packages{}
-	file := filepath.Join(path, "go.sum")
-	goSumContent, err := common.ReadAllFile(file)
-	if err != nil {
-		log.Println(err)
-		return ecosystem.NewProject(path, EcosystemName, pkgs)
-	}
-
-	lines := strings.Split(string(goSumContent), common.DetectLineEnding(file))
+func parsePackagesFromGoSumContent(content string) es.Packages {
+	pkgs := es.Packages{}
+	lines := strings.Split(content, common.DetectStringLineEnding(content))
 
 	for _, line := range lines {
 		if line == "" {
@@ -31,8 +22,7 @@ func NewProjectFromGoSum(path string) ecosystem.Project {
 
 		name := parts[0]
 		version := parts[1][1:]
-		pkgs.Append(ecosystem.NewDefaultPackage(name, version, ""))
+		pkgs.Append(es.NewDefaultPackage(name, version, ""))
 	}
-
-	return ecosystem.NewProject(path, EcosystemName, pkgs)
+	return pkgs
 }

core/ecosystem/golang/project.go

@@ -0,0 +1,7 @@
+package golang
+
+import "cvepack/core/ecosystem"
+
+func NewProject(source ecosystem.Source, packages []ecosystem.Package) ecosystem.Project {
+	return ecosystem.NewProject(source.Value, EcosystemName, packages)
+}