Merge pull request #189 from khalifapro/npm-yarn-bug-fixing

Npm yarn bug fixing
opensbom-generator · Jun 28, 2021 · ecd5ebc · ecd5ebc
2 parents 71918e6 + 1747d77
commit ecd5ebc
Show file tree

Hide file tree

Showing 5 changed files with 185 additions and 58 deletions.
diff --git a/internal/format/format.go b/internal/format/format.go
@@ -94,7 +94,7 @@ func (f *Format) Render() error {
 	if len(otherLicenses) > 0 {
 		file.WriteString("##### Non-standard license\n\n")
 		for lic := range otherLicenses {
-			file.WriteString(fmt.Sprintf("LicenseID: %s\n", lic))
+			file.WriteString(fmt.Sprintf("LicenseID: LicenseRef-%s\n", lic))
 			file.WriteString(fmt.Sprintf("ExtractedText: %s\n", otherLicenses[lic].ExtractedText))
 			file.WriteString(fmt.Sprintf("LicenseName: %s\n", otherLicenses[lic].Name))
 			file.WriteString(fmt.Sprintf("LicenseComment: %s\n\n", otherLicenses[lic].Comments))

diff --git a/internal/modules/npm/handler.go b/internal/modules/npm/handler.go
@@ -97,6 +97,8 @@ func (m *npm) GetRootModule(path string) (*models.Module, error) {
 	}
 	mod := &models.Module{}
 
+	splitedPath := strings.Split(path, "/")
+	mod.Name = splitedPath[len(splitedPath)-1]
 	if pkResult["name"] != nil {
 		mod.Name = pkResult["name"].(string)
 	}
@@ -183,57 +185,70 @@ func (m *npm) buildDependencies(path string, deps map[string]interface{}) ([]mod
 	for k, v := range rootDeps {
 		de.Modules[k] = v
 	}
-
 	modules = append(modules, *de)
-	for key, dd := range deps {
-		d := dd.(map[string]interface{})
-		var mod models.Module
-		mod.Version = d["version"].(string)
-		mod.Name = strings.TrimPrefix(key, "@")
-
-		r := ""
-		if d["resolved"] != nil {
-			r = d["resolved"].(string)
-			if strings.Contains(r, npmRegistry) {
-				mod.Supplier.Name = "NOASSERTION"
+
+	allDeps := appendNestedDependencies(deps)
+	for key, dd := range allDeps {
+		depName := strings.TrimPrefix(key, "@")
+		for nkey := range dd {
+			var mod models.Module
+			d := dd[nkey].(map[string]interface{})
+			mod.Version = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(nkey, "^"), "~"), ">"),"="))
+			mod.Version = strings.Split(mod.Version, " ")[0]
+			mod.Name = depName
+
+			r := ""
+			if d["resolved"] != nil {
+				r = d["resolved"].(string)
+				if strings.Contains(r, npmRegistry) {
+					mod.Supplier.Name = "NOASSERTION"
+				}
 			}
-		}
-		mod.PackageURL = helper.RemoveURLProtocol(r)
-		h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)))
-		mod.CheckSum = &models.CheckSum{
-			Algorithm: "SHA256",
-			Value:     h,
-		}
-		mod.Copyright = getCopyright(filepath.Join(path, m.metadata.ModulePath[0], key))
-		modLic, err := helper.GetLicenses(filepath.Join(path, m.metadata.ModulePath[0], key))
-		if err != nil {
-			continue
-		}
-		mod.LicenseDeclared = helper.BuildLicenseDeclared(modLic.ID)
-		mod.LicenseConcluded = helper.BuildLicenseConcluded(modLic.ID)
-		mod.CommentsLicense = modLic.Comments
-		if !helper.LicenseSPDXExists(modLic.ID) {
-			mod.OtherLicense = append(mod.OtherLicense, modLic)
-		}
-		mod.Modules = map[string]*models.Module{}
-		if d["requires"] != nil {
-			modDeps := d["requires"].(map[string]interface{})
-			deps := getPackageDependencies(modDeps, "requires")
-			for k, v := range deps {
-				mod.Modules[k] = v
+
+			mod.PackageURL = getPackageHomepage(filepath.Join(path, m.metadata.ModulePath[0], key, m.metadata.Manifest[0]))
+			mod.PackageDownloadLocation = r
+			h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)))
+			mod.CheckSum = &models.CheckSum{
+				Algorithm: "SHA256",
+				Value:     h,
 			}
-		}
 
-		if d["dependencies"] != nil {
-			modDeps := d["dependencies"].(map[string]interface{})
-			deps := getPackageDependencies(modDeps, "dependencies")
-			for k, v := range deps {
-				mod.Modules[k] = v
+			mod.Copyright = getCopyright(filepath.Join(path, m.metadata.ModulePath[0], key))
+			mod.Modules = map[string]*models.Module{}
+			if dd["requires"] != nil {
+				modDeps := dd["requires"].(map[string]interface{})
+				deps := getPackageDependencies(modDeps, "requires")
+				for k, v := range deps {
+					mod.Modules[k] = v
+				}
 			}
+
+			if dd["dependencies"] != nil {
+				modDeps := dd["dependencies"].(map[string]interface{})
+				deps := getPackageDependencies(modDeps, "dependencies")
+				for k, v := range deps {
+					mod.Modules[k] = v
+				}
+			}
+
+			modLic, err := helper.GetLicenses(filepath.Join(path, m.metadata.ModulePath[0], key))
+			if err != nil {
+				modules = append(modules, mod)
+				continue
+			}
+			mod.LicenseDeclared = helper.BuildLicenseDeclared(modLic.ID)
+			mod.LicenseConcluded = helper.BuildLicenseConcluded(modLic.ID)
+			mod.CommentsLicense = modLic.Comments
+			if !helper.LicenseSPDXExists(modLic.ID) {
+				mod.OtherLicense = append(mod.OtherLicense, modLic)
+			}
+
+			modules = append(modules, mod)
+
 		}
 
-		modules = append(modules, mod)
 	}
+
 	return modules, nil
 }
 
@@ -291,3 +306,63 @@ func getPackageDependencies(modDeps map[string]interface{}, t string) map[string
 	}
 	return m
 }
+
+func getPackageHomepage(path string) string {
+	r := reader.New(path)
+	pkResult, err := r.ReadJson()
+	if err != nil {
+		return ""
+	}
+	if pkResult["homepage"] != nil {
+		return helper.RemoveURLProtocol(pkResult["homepage"].(string))
+	}
+	return ""
+}
+
+func appendNestedDependencies(deps map[string]interface{}) map[string]map[string]interface{} {
+	allDeps := make(map[string]map[string]interface{})
+	for k, v := range deps {
+		mainDeps := make(map[string]interface{})
+
+		if allDeps[k] != nil {
+			mainDeps = allDeps[k]
+		}
+		mainDeps[v.(map[string]interface{})["version"].(string)] = v
+		allDeps[k] = mainDeps
+
+		if r, ok := v.(map[string]interface{})["requires"]; ok {
+			appendRequired(r, allDeps)
+		}
+
+		if d, ok := v.(map[string]interface{})["dependencies"]; ok {
+			appendDependencies(d, allDeps)
+		}
+
+	}
+	return allDeps
+}
+
+func appendDependencies(d interface{}, allDeps map[string]map[string]interface{} ) {
+	for dk, dv := range d.(map[string]interface{}) {
+		m := allDeps[dk]
+		if m == nil {
+			m = make(map[string]interface{})
+		}
+		m[dv.(map[string]interface{})["version"].(string)] = dv.(map[string]interface{})
+		allDeps[dk] = m
+	}
+}
+
+func appendRequired(r interface{}, allDeps map[string]map[string]interface{}) {
+	for rk, rv := range r.(map[string]interface{}) {
+		if rv.(string) == "*" {
+			continue
+		}
+		nestedDeps := allDeps[rk]
+		if nestedDeps == nil {
+			nestedDeps = make(map[string]interface{})
+		}
+		nestedDeps[rv.(string)] = map[string]interface{}{"version": rv}
+		allDeps[rk] = nestedDeps
+	}
+}
diff --git a/internal/modules/npm/handler_test.go b/internal/modules/npm/handler_test.go
@@ -108,7 +108,7 @@ func TestListAllModules(t *testing.T) {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)) )
 
 			assert.Equal(t, "10.11.0", mod.Version)
-			assert.Equal(t, "registry.npmjs.org/validator/-/validator-10.11.0.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) 2018 Chris O'Hara <cohara87@gmail.com>", mod.Copyright)
@@ -120,7 +120,7 @@ func TestListAllModules(t *testing.T) {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)))
 
 			assert.Equal(t, "2.2.16", mod.Version)
-			assert.Equal(t, "registry.npmjs.org/shortid/-/shortid-2.2.16.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.npmjs.org/shortid/-/shortid-2.2.16.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) Dylan Greene", mod.Copyright)
@@ -132,7 +132,7 @@ func TestListAllModules(t *testing.T) {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)) )
 
 			assert.Equal(t, "1.19.0", mod.Version)
-			assert.Equal(t, "registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) 2014 Jonathan Ong <me@jongleberry.com>", mod.Copyright)
@@ -143,7 +143,7 @@ func TestListAllModules(t *testing.T) {
 		if mod.Name == "bcryptjs" {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)) )
 			assert.Equal(t, "2.4.3", mod.Version)
-			assert.Equal(t, "registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) 2012 Nevins Bartolomeo <nevins.bartolomeo@gmail.com>", strings.TrimSpace(mod.Copyright))

diff --git a/internal/modules/yarn/handler.go b/internal/modules/yarn/handler.go
@@ -151,11 +151,12 @@ func (m *yarn) ListUsedModules(path string) ([]models.Module, error) {
 // ListModulesWithDeps return all info of installed modules
 func (m *yarn) ListModulesWithDeps(path string) ([]models.Module, error) {
 	deps, err := readLockFile(filepath.Join(path, lockFile))
+	allDeps := appendNestedDependencies(deps)
 	if err != nil {
 		return nil, err
 	}
 
-	return m.buildDependencies(path, deps)
+	return m.buildDependencies(path, allDeps)
 }
 
 func (m *yarn) buildDependencies(path string, deps []dependency) ([]models.Module, error) {
@@ -173,11 +174,11 @@ func (m *yarn) buildDependencies(path string, deps []dependency) ([]models.Modul
 	for _, d := range deps {
 		var mod models.Module
 		mod.Name = d.Name
-		mod.Version = d.Version
+		mod.Version = trimPrefixes(d.Version)
 		modules[0].Modules[d.Name] = &models.Module{
 			Name:     d.Name,
-			Version:  d.Version,
-			CheckSum: &models.CheckSum{Content: []byte(fmt.Sprintf("%s-%s", d.Name, d.Version))},
+			Version:  mod.Version,
+			CheckSum: &models.CheckSum{Content: []byte(fmt.Sprintf("%s-%s", d.Name, mod.Version))},
 		}
 		if len(d.Dependencies) != 0 {
 			mod.Modules = map[string]*models.Module{}
@@ -187,10 +188,11 @@ func (m *yarn) buildDependencies(path string, deps []dependency) ([]models.Modul
 				if name == "optionalDependencies:" {
 					continue
 				}
-				version := strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(strings.TrimSpace(ar[1]), "\"" ), "^"), "\"")
+
+				version := strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(ar[1]), "\"" ), "\"")
 				mod.Modules[name] = &models.Module{
 					Name:     name,
-					Version:  version,
+					Version:  trimPrefixes(version),
 					CheckSum: &models.CheckSum{Content: []byte(fmt.Sprintf("%s-%s", name, version))},
 				}
 			}
@@ -203,7 +205,8 @@ func (m *yarn) buildDependencies(path string, deps []dependency) ([]models.Modul
 			mod.Supplier.Name = "NOASSERTION"
 		}
 
-		mod.PackageURL = helper.RemoveURLProtocol(r)
+		mod.PackageURL = getPackageHomepage(filepath.Join(path, m.metadata.ModulePath[0], d.PkPath, m.metadata.Manifest[0]))
+		mod.PackageDownloadLocation = r
 		h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name)))
 		mod.CheckSum = &models.CheckSum{
 			Algorithm: "SHA256",
@@ -218,6 +221,7 @@ func (m *yarn) buildDependencies(path string, deps []dependency) ([]models.Modul
 
 		modLic, err := helper.GetLicenses(filepath.Join(path, m.metadata.ModulePath[0], d.PkPath ))
 		if err != nil {
+			modules = append(modules, mod)
 			continue
 		}
 		mod.LicenseDeclared = helper.BuildLicenseDeclared(modLic.ID)
@@ -328,3 +332,51 @@ func getCopyright(path string) string {
 
 	return ""
 }
+
+
+func getPackageHomepage(path string) string {
+	r := reader.New(path)
+	pkResult, err := r.ReadJson()
+	if err != nil {
+		return ""
+	}
+	if pkResult["homepage"] != nil {
+		return helper.RemoveURLProtocol(pkResult["homepage"].(string))
+	}
+	return ""
+}
+
+
+func trimPrefixes(s string) string{
+	t := strings.TrimPrefix(s, "^")
+	t = strings.TrimPrefix(t, "~")
+	t = strings.TrimPrefix(t, ">")
+	t = strings.TrimPrefix(t, "=")
+
+	t = strings.Split(t, " ")[0]
+	return t
+}
+
+
+func appendNestedDependencies(deps []dependency) []dependency{
+	allDeps := make([]dependency, 0)
+	for _,d := range deps {
+		allDeps = append(allDeps, d)
+		if len(d.Dependencies) > 0 {
+			for _,depD := range d.Dependencies {
+				ar := strings.Split(strings.TrimSpace(depD), " ")
+				name := strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(ar[0], "\""), "\""), "@")
+				if name == "optionalDependencies:" {
+					continue
+				}
+
+				version := strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(ar[1]), "\"" ), "\"")
+				if trimPrefixes(version) == "*"{
+					continue
+				}
+				allDeps = append(allDeps, dependency{Name: name, Version: trimPrefixes(version)})
+			}
+		}
+	}
+	return allDeps
+}
diff --git a/internal/modules/yarn/handler_test.go b/internal/modules/yarn/handler_test.go
@@ -99,7 +99,7 @@ func TestListAllModules(t *testing.T) {
 		if mod.Name == "axios" {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s", mod.Name))) )
 			assert.Equal(t, "0.19.2", mod.Version)
-			assert.Equal(t, "registry.yarnpkg.com/axios/-/axios-0.19.2.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) 2014-present Matt Zabriskie", mod.Copyright)
@@ -111,7 +111,7 @@ func TestListAllModules(t *testing.T) {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s", mod.Name))) )
 
 			assert.Equal(t, "16.14.0", mod.Version)
-			assert.Equal(t, "registry.yarnpkg.com/react/-/react-16.14.0.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.yarnpkg.com/react/-/react-16.14.0.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) Facebook, Inc. and its affiliates.", mod.Copyright)
@@ -123,7 +123,7 @@ func TestListAllModules(t *testing.T) {
 			h := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s", mod.Name))) )
 
 			assert.Equal(t, "16.14.0", mod.Version)
-			assert.Equal(t, "registry.yarnpkg.com/react-dom/-/react-dom-16.14.0.tgz", mod.PackageURL)
+			assert.Equal(t, "https://registry.yarnpkg.com/react-dom/-/react-dom-16.14.0.tgz", mod.PackageDownloadLocation)
 			assert.Equal(t, models.HashAlgorithm("SHA256"), mod.CheckSum.Algorithm)
 			assert.Equal(t, h, mod.CheckSum.Value)
 			assert.Equal(t, "Copyright (c) Facebook, Inc. and its affiliates.", mod.Copyright)