流程优化;修复一些逻辑问题;增加出错重试

.gitignore

@@ -2,6 +2,7 @@
 *.db
 .idea
 logs
+test
 .DS_Store
 # Test binary, built with `go test -c`
 *.test

cmd/main.go

@@ -15,6 +15,7 @@ func main() {
 	runner.ParseArguments()
 	go runner.Cyclic()
 	go runner.Run()
+	go runner.Retry()
 
 	web.Init()
 }

pkg/runner/clnoe.go

@@ -20,7 +20,8 @@ func GitClone(gurl string, name string) error {
 	err := cmd.Run()
 	_, errStr := string(stdout.Bytes()), string(stderr.Bytes())
 	if err != nil {
-		logging.Logger.Errorf("GitClone cmd.Run() failed with %s --  %s\n", err, errStr)
+		logging.Logger.Errorf("GitClone(%s) cmd.Run() failed with %s --  %s\n", gurl, err, errStr)
+
 		return err
 	}
 	return nil

pkg/runner/codeql.go

@@ -21,12 +21,15 @@ import (
 **/
 
 func Analyze(database string, name string, language string, qls []string) map[string]string {
-	if qls == nil {
-		if strings.ToLower(language) == "go" {
-			qls = QLFiles.GoQL
-		} else if strings.ToLower(language) == "java" {
-			qls = QLFiles.JavaQL
-		}
+	if language == "Go" {
+		qls = QLFiles.GoQL
+	} else if language == "Java" {
+		qls = QLFiles.JavaQL
+	}
+
+	if len(qls) == 0 {
+		logging.Logger.Debugln("qls = 0")
+		return nil
 	}
 
 	res := make(map[string]string)
@@ -49,10 +52,14 @@ func Analyze(database string, name string, language string, qls []string) map[st
 
 		lines := utils.LoadFile(fileName)
 
+		if len(lines) == 0 {
+			continue
+		}
+
 		var result string
 
-		for _, i := range lines {
-			result += i
+		for _, line := range lines {
+			result += line
 		}
 		res[fileName] = result
 
@@ -72,10 +79,7 @@ func Analyze(database string, name string, language string, qls []string) map[st
 }
 
 // CreateDb 拉取仓库，本地创建数据库
-func CreateDb(gurl string, res *githubRes, name string) string {
-	if !utils.StringInSlice(res.Language, Languages) {
-		return ""
-	}
+func CreateDb(gurl, languages string) string {
 	dbName := utils.GetName(gurl)
 	err := GitClone(gurl, dbName)
 
@@ -85,7 +89,7 @@ func CreateDb(gurl string, res *githubRes, name string) string {
 	}
 
 	// todo 批量跑就抽风，导致有的项目无法生成数据库 "There's no CodeQL extractor named 'Go' installed."
-	cmd := exec.Command("codeql", "database", "create", DirNames.DbDir+dbName, "-s", DirNames.GithubDir+dbName, "--language="+strings.ToLower(res.Language), "--overwrite")
+	cmd := exec.Command("codeql", "database", "create", DirNames.DbDir+dbName, "-s", DirNames.GithubDir+dbName, "--language="+strings.ToLower(languages), "--overwrite")
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout = &stdout // 标准输出
 	cmd.Stderr = &stderr // 标准错误
@@ -98,12 +102,27 @@ func CreateDb(gurl string, res *githubRes, name string) string {
 
 	// 很奇怪，有的生成数据库不是在项目目录下，而是在第二级目录下
 	dbPath := filepath.Dir(path.Join(utils.CodeqlDb(DirNames.DbDir+dbName), "*"))
+	logging.Logger.Debugln(gurl, " CreateDb success")
 	return dbPath
 }
 
 // UpdateRule 每天拉取一下官方仓库，更新规则
 func UpdateRule() {
 	if Option.Path != "" {
-		utils.RunGitCommand(Option.Path, "git", "pull")
+		_, err := utils.RunGitCommand(Option.Path, "git", "pull")
+		record := db.Record{
+			Project: "CodeQL Rules",
+			Url:     "CodeQL Rules",
+			Color:   "success",
+			Title:   "CodeQL Rules",
+			Msg:     "CodeQL Rules 更新成功",
+		}
+
+		if err != nil {
+			record.Color = "danger"
+			record.Msg = fmt.Sprintf("CodeQL Rules 更新失败, %s", err.Error())
+		}
+
+		db.AddRecord(record)
 	}
 }

pkg/runner/cyclic.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"Yi/pkg/db"
 	"fmt"
+	"github.com/thoas/go-funk"
 	"os"
 	"sync"
 	"time"
@@ -18,7 +19,12 @@ func Cyclic() {
 	for {
 		// todo 不够优雅，万一监控的项目过多，导致一天还没执行完呢
 		// 等待24小时后再循环执行
-		time.Sleep(24 * 60 * time.Minute)
+		if !Option.RunNow {
+			time.Sleep(24 * 60 * time.Minute)
+		}
+
+		Option.RunNow = false
+
 		// 更新规则库
 		UpdateRule()
 
@@ -43,7 +49,7 @@ func Cyclic() {
 		limit := make(chan bool, Option.Thread)
 
 		for _, p := range projects {
-			if p.DBPath == "" {
+			if p.DBPath == "" || !funk.Contains(Languages, p.Language) {
 				continue
 			}
 			wg.Add(1)
@@ -54,27 +60,34 @@ func Cyclic() {
 					wg.Done()
 				}()
 
-				// 更新了才会去生成数据库
-				update, dbPath, pushedAt := CheckUpdate(project.Url, project.PushedAt, project.Project)
+				// 说明 之前运行失败了, 再尝试一次执行
+				if project.Count == 0 {
+					Exec(project, nil)
+				} else {
+					// 更新了才会去生成数据库
+					update, dbPath, pushedAt := CheckUpdate(project)
 
-				if !update {
-					return
-				}
+					if !update {
+						return
+					}
 
-				count++
-				project.DBPath = dbPath
-				project.PushedAt = pushedAt
+					count++
+					project.DBPath = dbPath
+					project.PushedAt = pushedAt
 
-				db.UpdateProject(project.Id, project)
-
-				Exec(project, nil)
+					db.UpdateProject(project.Id, project)
+					Exec(project, nil)
+				}
 			}(p)
 
 		}
 
 		wg.Wait()
 		close(limit)
 
+		// 全部运行完后，开始对出错的项目进行重试
+		IsRetry = true
+
 		record := db.Record{
 			Color: "primary",
 			Title: "新一轮扫描",

pkg/runner/default.go

@@ -43,10 +43,8 @@ go_ql:
   - go/ql/src/Security/CWE-643/XPathInjection.ql
   # 硬编码认证信息
   - go/ql/src/Security/CWE-798/HardcodedCredentials.ql	
-  # 重定向
-  - go/ql/src/myRules/UrlRedirect.ql
-  # 任意文件读取
-  - go/ql/src/myRules/ReadFile.ql
+  # 自定义规则
+  - go/ql/src/myRules/
 java_ql:
   # 路径问题
   - java/ql/src/Security/CWE/CWE-022/TaintedPath.ql