update dirbrute module to support 302 redirect

godspeedcurry · Jul 3, 2024 · b6c2ed9 · b6c2ed9
1 parent af7e64b
commit b6c2ed9
Show file tree

Hide file tree

Showing 11 changed files with 106 additions and 68 deletions.
diff --git a/cmd/dirbrute.go b/cmd/dirbrute.go
@@ -49,7 +49,7 @@ func (o *DirbruteOptions) run() {
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetAutoWrapText(false)
 
-	table.SetHeader([]string{"Url", "Title", "Finger", "Content-Type", "StatusCode", "Length", "SimHash", "Keyword"})
+	table.SetHeader(common.TableHeader)
 
 	for _, line := range targetUrlList {
 		for _, dir := range targetDirList {

diff --git a/cmd/port.go b/cmd/port.go
@@ -16,6 +16,7 @@ type PortOptions struct {
 	IpRange         string
 	IpRangeFile     string
 	PortRange       string
+	TopPorts        int
 	useAllProbes    bool
 	nullProbeOnly   bool
 	scanSendTimeout int
@@ -42,6 +43,7 @@ func init() {
 	ipCmd.PersistentFlags().StringVarP(&portOptions.IpRangeFile, "host-file", "I", "", "your ip list file")
 
 	ipCmd.PersistentFlags().StringVarP(&portOptions.PortRange, "port", "p", strings.Join(common.DefaultPorts, ","), "your port list")
+	ipCmd.PersistentFlags().IntVarP(&portOptions.TopPorts, "top", "", 0, "top ports to scan, default is 500")
 
 	ipCmd.PersistentFlags().IntVarP(&portOptions.scanSendTimeout, "scan-send-timeout", "s", 5, "Set connection send timeout in seconds")
 	ipCmd.PersistentFlags().IntVarP(&portOptions.scanReadTimeout, "scan-read-timeout", "r", 5, "Set connection read timeout in seconds")
@@ -61,6 +63,9 @@ func init() {
 	viper.BindPFlag("port", ipCmd.PersistentFlags().Lookup("port"))
 	viper.SetDefault("port", "")
 
+	viper.BindPFlag("top", ipCmd.PersistentFlags().Lookup("top"))
+	viper.SetDefault("top", 0)
+
 	viper.BindPFlag("scan-rarity", ipCmd.PersistentFlags().Lookup("scan-rarity"))
 	viper.SetDefault("scan-rarity", 5)
 
@@ -82,8 +87,19 @@ func (o *PortOptions) run() {
 		ips := utils.FileReadLine(portOptions.IpRangeFile)
 		portOptions.IpRange = strings.Join(ips, ",")
 	} else if portOptions.IpRange == "" {
-		utils.Error("Please provide ip range or ip range file")
+		utils.Info("Please provide ip range or ip range file")
 		return
 	}
-	utils.PortScan(portOptions.IpRange, portOptions.PortRange)
+	if portOptions.TopPorts != 0 {
+		if portOptions.TopPorts > 20000 {
+			utils.Info("We do not have more than top 20000 ports, please choose a smaller number, or just scan all ports use `-p 0-65535`")
+			return
+		} else {
+			utils.PortScan(portOptions.IpRange, strings.Join(strings.Split(common.AllPorts, ",")[0:portOptions.TopPorts], ","))
+		}
+
+	} else {
+		utils.PortScan(portOptions.IpRange, portOptions.PortRange)
+	}
+
 }
diff --git a/common/config.go b/common/config.go
@@ -1,5 +1,9 @@
 package common
 
+import (
+	_ "embed"
+)
+
 var Userdict = map[string][]string{
 	"ftp":        {"ftp", "admin", "www", "web", "root", "db", "wwwroot", "data", "test", "administrator", "anonymous"},
 	"mysql":      {"root", "mysql"},
@@ -18,7 +22,7 @@ var Patterns = []string{"@", "_", "#", ""}
 var Passwords = []string{"!@#QWEASD", "!@#QWEASDZXC", "!QAZ2wsx", "0", "00000", "00001", "000000", "00000000", "1", "111111", "12", "123", "123123", "123321", "123456", "123!@#qwe", "123!@#asd", "123!@#zxc", "123456!a", "1234567", "12345678", "123456789", "1234567890", "123456~a", "123654", "123qwe", "123qwe!@#", "1q2w#E$R", "1q2w3e", "1q2w3e4r", "1qaz!QAZ", "1qaz2wsx", "1qaz2wsx3edc", "1qaz@WSX", "1qaz@wsx#edc", "2wsx@WSX", "654123", "654321", "666666", "8888888", "88888888", "a11111", "a123123", "a12345", "a123456", "a123456.", "A123456s!", "Aa123123", "Aa1234", "Aa1234.", "Aa12345", "Aa12345.", "Aa123456", "Aa123456!", "Aa123456789", "abc123", "abc@123", "abc123456", "admin", "admin01", "admin123", "admin123!@#", "admin@123", "Admin@123", "Change_Me", "Change_Me123", "Charge123", "manager", "P@ssw0rd", "P@ssw0rd!", "pass123", "pass@123", "Passw0rd", "password", "qazwsxedc", "qwe123", "qwe123!@#", "root", "sa123456", "shell", "sysadmin", "system", "talent", "test", "test01", "test123", "toor", "admin0", "admin1", "admin2", "adminadmin", "Test@123", "Abd@1234"}
 
 var DirList = []string{
-	"..;/actuator/env", "..;/api-docs", "..;/env", "..;/swagger-ui.html", "..;/v2/api-docs", ".DS_Store", ".git/config", ".git/HEAD", ".git/index", ".svn", "/", "actuator", "actuator/env", "actuator;.js", "admin", "api", "api-docs", "api-docs/", "api-docs/index.html", "api/", "api/actuator", "api/index.html", "api/swagger-resources", "api/swagger-ui.html", "api/v2/api-docs", "apidocs/", "apidocs/index.html", "core/auth/login", "docs/", "docs/index.html", "env", "geoserver/index.html", "jeecg-boot", "mappings", "nacos", "nacos/#/", "service", "services", "site.tar.gz", "swagger-resources", "swagger-ui.html", "swagger/", "swagger/index.html", "v2/api-docs", "web.tar.gz", "www.tar.gz", "xxl-job-admin", "version", "log", "metrics", "cluster", "node", "api/v1/nodes", "pods", "v2/keys",
+	"..;/actuator/env", "..;/api-docs", "..;/env", "..;/swagger-ui.html", "..;/v2/api-docs", ".DS_Store", ".git/config", ".git/HEAD", ".git/index", ".svn", "/", "actuator", "actuator/env", "actuator;.js", "admin", "api", "api-docs", "api-docs/", "api-docs/index.html", "api/", "api/actuator", "api/index.html", "api/swagger-resources", "api/swagger-ui.html", "api/v2/api-docs", "apidocs/", "apidocs/index.html", "core/auth/login", "docs/", "docs/index.html", "env", "geoserver/index.html", "jeecg-boot", "mappings", "nacos", "nacos/", "nacos/#/", "service", "services", "site.tar.gz", "swagger-resources", "swagger-ui.html", "swagger/", "swagger/index.html", "v2/api-docs", "web.tar.gz", "www.tar.gz", "xxl-job-admin", "version", "log", "metrics", "cluster", "node", "api/v1/nodes", "pods", "v2/keys",
 	"..;/actuator", "..;/..;/actuator", "..;/..;/..;/actuator", "..;/..;/..;/..;/actuator", "..;/..;/..;/..;/..;/actuator", "..;/..;/..;/..;/..;/..;/..;/..;/actuator",
 }
 
@@ -52,22 +56,31 @@ var PORTList = map[string]int{
 var IsSave = true
 var MostSensitiveWebPort = "80,443,8080"
 
-var DefaultPorts = []string{
-	"1000", "10000-10030", "10050", "1010", "10206", "10250", "10253-10255", "1043", "10443", "1080-1082", "1099", "111", "1118", "11211",
-	"1194", "12018", "123", "1234", "12345", "135-139", "138", "1433", "14390", "1443", "1516", "1521", "16080", "1701", "1723", "179",
-	"18000-18004", "18008", "18080-18085", "18088", "18090", "18098", "181", "1888", "1947", "2000", "20000", "2008", "2016", "2020", "2024",
-	"20443", "2049", "20720", "20880", "2100", "21000", "21443", "21500-21502", "2181", "2374-2376", "2379", "23743", "23", "25000-25005",
-	"253", "26000-26010", "27017-27018", "28018", "2869", "3000", "3008", "30443", "3128", "31943", "31945", "32", "33038", "3306-3308",
-	"33060-33065", "3333", "3389", "34987", "37445", "38443", "39443", "389", "4500", "4789", "4848", "4899", "49336", "49593", "2", "50", "500",
-	"5000", "50050-50051", "5050", "5065", "514", "5228", "5353", "5355", "5357", "5432", "54321", "54303", "5555", "5632", "56610", "5678",
-	"5985", "60443", "6060", "6080", "61227", "6379", "6443", "6648", "6783", "6881", "7070-7071", "7074", "7078", "7080", "7088", "7200", "768",
-	"7680", "7687-7688", "7808", "7890", "79", "8000", "8000-8019", "800-801", "801", "8020", "8028", "8030", "8038", "8042", "8044", "8046",
-	"8048", "8053", "8060", "8069-8070", "808", "8080", "8080-8099", "8089", "8099", "8100-8101", "8108", "8118", "8161", "8172", "8180-8181",
-	"8200", "8222", "8244", "8258", "8280", "8288", "8300", "8360", "8443", "8448", "8484", "8472", "880", "8800", "8834", "8838", "8848",
-	"8858", "886-889", "8868", "8879", "8880-8881", "8888-8890", "8983", "8989", "9000-9010", "9043", "9060", "9080-9102", "9198", "9200",
-	"9300", "9443", "9448", "9786", "9800", "9981", "9986", "9988", "9998-9999",
+// database
+// cloud
+// web
+// top 100
+// top 500
+// top 1000
+// top 5000
+// top 10000
+// top 20000
+var WebPorts = []string{
+	"21", "22", "25", "80-88", "443",
+	"10000-10030", "10250", "10443", "18080-18085", "20443", "21443", "30443",
+	"3306-3308", "3389", "38443", "39443", "5000", "50050-50051", "5432", "54321",
+	"5985", "60443", "6443", "8000-8019", "8020", "8080-8099", "8443", "8448",
+	"8880-8881", "8888-8890", "9000-9010", "9043", "9060", "9080-9102", "9200",
+	"9443", "9998-9999",
 }
 
+//go:embed ports_20000.txt
+var customProbes string
+
+var AllPorts = customProbes
+var DefaultPorts = []string{"80", "443", "2083", "8080", "7547", "2095", "22", "2078", "2096", "2087", "2077", "8443", "888", "2082", "5060", "2086", "8000", "8888", "161", "21", "8880", "53", "8089", "2052", "554", "30005", "8081", "2053", "52230", "2080", "4567", "8008", "1701", "2079", "3389", "58000", "500", "8088", "1723", "81", "2000", "123", "8085", "25", "37777", "23", "49152", "2091", "5985", "51005", "9000", "1024", "3306", "111", "5000", "7080", "8082", "47001", "7170", "8001", "6881", "49154", "49153", "139", "88", "50001", "445", "1194", "9090", "5001", "135", "1025", "49155", "8291", "110", "50995", "49665", "14440", "587", "14430", "9020", "9080", "3000", "50805", "2222", "143", "520", "993", "4433", "30010", "8090", "9200", "50996", "51001", "8015", "50999", "995", "50997", "49667", "8002", "50998", "51000", "465", "7000", "51003", "51002", "20002", "82", "51004", "1717", "49666", "8083", "19000", "49156", "5357", "49664", "9100", "8084", "7777", "8887", "9999", "10000", "49668", "5678", "3128", "52869", "6467", "6466", "10250", "8181", "9001", "49157", "58603", "9530", "37443", "10443", "444", "1026", "9010", "10001", "137", "8086", "6443", "49669", "2107", "8999", "60000", "20201", "2105", "2103", "4443", "85", "1080", "9443", "20000", "51007", "55555", "8020", "18080", "12121", "17000", "60002", "7001", "5432", "5555", "8009", "49158", "3001", "9527", "5006", "32400", "9091", "7848", "8899", "40000", "9876", "9305", "8010", "1433", "1900", "7443", "2525", "12345", "8444", "90", "10002", "6000", "1027", "50777", "8172", "10101", "8099", "8889", "9307", "9304", "4430", "6060", "5353", "8800", "8200", "50000", "2121", "4343", "9306", "9303", "83", "9002", "4444", "1883", "5523", "9003", "1500", "9998", "5900", "6379", "2323", "7081", "5683", "30006", "3333", "52200", "4040", "515", "6363", "8728", "1234", "7070", "43999", "6699", "631", "2223", "8087", "10010", "4000", "9009", "2601", "6001", "10022", "8100", "2049", "49502", "7005", "7548", "800", "3307", "541", "50580", "119", "20202", "8003", "49501", "8069", "8989", "5061", "179", "12350", "65004", "1000", "8282", "51200", "10011", "5005", "49159", "84", "6264", "3479", "3005", "646", "26", "10005", "8091", "12349", "42235", "9500", "2443", "873", "27017", "5986", "8445", "8006", "4911", "3002", "8096", "9012", "7003", "7004", "30003", "7010", "2379", "6666", "22222", "5222", "1028", "7002", "3443", "9004", "9092", "9800", "9101", "8159", "18018", "5431", "808", "9600", "999", "8005", "8787", "24442", "89", "10080", "5002", "9013", "9093", "3030", "8061", "602", "9021", "43080", "3003", "9099", "10020", "8990", "30000", "3006", "8580", "1443", "9005", "28080", "5007", "7778", "50011", "9191", "8881", "86", "8004", "8058", "3050", "8686", "50002", "38520", "8022", "8991", "9109", "6789", "91", "18443", "8383", "9030", "7071", "9444", "7800", "18017", "1201", "9103", "9088", "49161", "5080", "3702", "8123", "8060", "843", "18888", "7011", "8050", "990", "8070", "3031", "8180", "4848", "1029", "2404", "8016", "12380", "8043", "1302", "19080", "3010", "4434", "6005", "60001", "8866", "8011", "8765", "4500", "4190", "7676", "30001", "5672", "9988", "4431", "9089", "6008", "52931", "1688", "3008", "6080", "9007", "15672", "8014", "15000", "10003", "7050", "8883", "5500", "8092", "8222", "9102", "5090", "9081", "9085", "20001", "8554", "9801", "9105", "9094", "19999", "6002", "8012", "9008", "9900", "5050", "50050", "5400", "6380", "8101", "8098", "42443", "3080", "2200", "3004", "2090", "16001", "5443", "40005", "8530", "30004", "3299", "9098", "7100", "9212", "113", "3400", "98", "9062", "7500", "21242", "2196", "1935", "11001", "10009", "44444", "4800", "7999", "8023", "9095", "9991", "9663", "9308", "7019", "7020", "25565", "15001", "666", "548", "6036", "3100", "9553", "9082", "60443", "5569", "10243", "50100", "9119", "9143", "9040", "9014", "21300", "8315", "5600", "7700", "20080", "99", "2332", "8585", "9201", "8025", "9019", "5601", "2600", "8097", "14443", "50012", "12588", "8500", "16443", "30021", "7013", "8885", "4321", "9083"}
+
+var TableHeader = []string{"Url", "Title", "Finger", "Content-Type", "StatusCode", "location", "Length", "SimHash", "Keyword"}
 var SuffixTop = []string{
 	"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
 	"00", "000", "0000", "00000", "000000", "01", "001", "02", "03",

diff --git a/utils/dirbrute.go b/utils/dirbrute.go
@@ -32,9 +32,9 @@ func DirBrute(baseUrl string, dir string) []string {
 		return []string{}
 	}
 	fullURL := baseURL.ResolveReference(&url.URL{Path: path.Join(baseURL.Path, dir)})
-	finger, _, title, contentType, respBody, statusCode := FingerScan(fullURL.String(), http.MethodGet)
-	if statusCode == 200 || statusCode == 500 {
-		result = CheckFinger(finger, title, fullURL.String(), contentType, respBody, statusCode)
+	finger, _, title, contentType, location, respBody, statusCode := FingerScan(fullURL.String(), http.MethodGet, false)
+	if statusCode == 200 || statusCode == 500 || statusCode == 302 || statusCode == 301 {
+		result = CheckFinger(finger, title, fullURL.String(), contentType, location, respBody, statusCode)
 	}
 	if len(result) > 0 {
 		WriteToCsv("dirbrute.csv", result)

diff --git a/utils/ehole.json b/utils/ehole.json
@@ -136937,14 +136937,6 @@
                 "ColdFusion.Ajax"
             ]
         },
-        {
-            "cms": "Adobe ColdFusion",
-            "method": "keyword",
-            "location": "body",
-            "keyword": [
-                "cdm"
-            ]
-        },
         {
             "cms": "Jirafe",
             "method": "keyword",

diff --git a/utils/finger.go b/utils/finger.go
@@ -173,7 +173,8 @@ func ImportantApiJudge(ApiResult string, Url string) {
 }
 
 func parseVueUrl(Url string, RootPath string, doc string, filename string) {
-	ApiReg := regexp.MustCompile(`["'](?P<path>/[\w/\-\|_=@\?\:]+?)["']`)
+	quote := "['\"`]"
+	ApiReg := regexp.MustCompile(quote + `[\w\$\{\}]*(?P<path>/[\w/\-\|_=@\?\:]+?)` + quote)
 
 	ApiResultTuple := ApiReg.FindAllStringSubmatch(strings.ReplaceAll(doc, "\\", ""), -1)
 	ApiResult := []string{}
@@ -251,7 +252,7 @@ func parseVueUrl(Url string, RootPath string, doc string, filename string) {
 		}
 		if _, ok := sensitiveUrl.Load(Url); !ok {
 			sensitiveUrl.Store(Url, true)
-			SensitiveInfoCollect(Url, doc)
+			SensitiveInfoCollect(Url, doc, filename)
 		}
 	}
 
@@ -300,7 +301,7 @@ func Spider(RootPath string, Url string, depth int, filename string, myMap mapse
 		}
 		if _, ok := sensitiveUrl.Load(Url); !ok {
 			sensitiveUrl.Store(Url, true)
-			SensitiveInfoCollect(Url, html)
+			SensitiveInfoCollect(Url, html, filename)
 		}
 
 		// a, link 标签
@@ -322,7 +323,6 @@ func Spider(RootPath string, Url string, depth int, filename string, myMap mapse
 			}
 			normalizeUrl := Normalize(src, RootPath)
 			if normalizeUrl != "" && !myMap.Contains(normalizeUrl) {
-
 				Spider(RootPath, normalizeUrl, depth-1, filename, myMap)
 			}
 		})
@@ -472,10 +472,10 @@ func PrintFinger(Url string, Depth int) {
 	// 首页
 	FirstUrl := RootPath + Host.Path
 
-	finger, server, title, contentType, respBody, statusCode := FingerScan(FirstUrl, http.MethodGet)
+	finger, server, title, contentType, _, respBody, statusCode := FingerScan(FirstUrl, http.MethodGet, true)
 
 	if statusCode != -1 {
-		result := CheckFinger(finger, title, Url, contentType, respBody, statusCode)
+		result := CheckFinger(finger, title, Url, contentType, "", respBody, statusCode)
 		if len(result) > 0 {
 			WriteToCsv("finger.csv", result)
 		}
@@ -484,9 +484,9 @@ func PrintFinger(Url string, Depth int) {
 
 	// 构造404 + POST
 	SecondUrl := RootPath + "/xxxxxx"
-	finger, server, title, contentType, respBody, statusCode = FingerScan(SecondUrl, http.MethodPost)
+	finger, server, title, contentType, _, respBody, statusCode = FingerScan(SecondUrl, http.MethodPost, true)
 	if statusCode != -1 {
-		result := CheckFinger(finger, title, Url, contentType, respBody, statusCode)
+		result := CheckFinger(finger, title, Url, contentType, "", respBody, statusCode)
 		if len(result) > 0 {
 			WriteToCsv("finger.csv", result)
 		}

diff --git a/utils/matchrule.go b/utils/matchrule.go
@@ -127,45 +127,60 @@ func preprocessAndEvaluate(input string, context map[string]string) (bool, error
 	return result.(bool), nil
 }
 
-func FingerScan(url string, method string) (string, string, string, string, []byte, int) {
+func FingerScan(url string, method string, followRedirect bool) (string, string, string, string, string, []byte, int) {
 	if !isValidUrl(url) {
-		return common.NoFinger, "", "", "", nil, -1
+		return common.NoFinger, "", "", "", "", nil, -1
 	}
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
-		Fatal("%s", err)
-		return common.NoFinger, "", "", "", nil, -1
+		Fatal("%s %s xxx", url, err)
+		return common.NoFinger, "", "", "", "", nil, -1
 	}
 	req.Header.Set("User-Agent", viper.GetString("DefaultUA"))
 	req.Header.Set("Cookie", "rememberMe=me")
+	if !followRedirect {
+		Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+
 	resp, err := Client.Do(req)
 	if err != nil {
-		Fatal("%s", err)
-		return common.NoFinger, "", "", "", nil, -1
+		Fatal("%s %s create request failed", url, err)
+		return common.NoFinger, "", "", "", "", nil, -1
 	}
 	defer resp.Body.Close()
+	ServerValue := resp.Header["Server"]
+	retServerValue := ""
+	if len(ServerValue) != 0 {
+		retServerValue = ServerValue[0]
+	}
+
+	if resp.StatusCode == 301 || resp.StatusCode == 302 {
+		return common.NoFinger, retServerValue, "", resp.Header.Get("Content-Type"), resp.Header.Get("Location"), nil, resp.StatusCode
+	}
 	headers := MapToJson(resp.Header)
 
 	var config Packjson
 
 	err = json.Unmarshal([]byte(eholeJson), &config)
 	if err != nil {
-		Fatal("%s", err)
-		return common.NoFinger, "", "", "", nil, -1
+		Fatal("%s %s unmarshal failed", url, err)
+		return common.NoFinger, "", "", "", "", nil, -1
 	}
 	var cms []string
 	bodyBytes, _ := io.ReadAll(resp.Body)
 	_, contentType, _ := charset.DetermineEncoding(bodyBytes, resp.Header.Get("Content-Type"))
 	reader, err := charset.NewReader(bytes.NewBuffer(bodyBytes), contentType)
 	if err != nil {
-		Fatal("%s", err)
-		return common.NoFinger, "", "", "", nil, -1
+		Fatal("%s %s %s", url, err, contentType)
+		return common.NoFinger, "", "", "", "", nil, -1
 	}
 	doc, err := goquery.NewDocumentFromReader(reader)
 
 	if err != nil {
-		Fatal("%s", err)
-		return common.NoFinger, "", "", "", nil, -1
+		Fatal("%s %s", url, err)
+		return common.NoFinger, "", "", "", "", nil, -1
 	}
 
 	// 查找标题元素并获取内容
@@ -201,10 +216,6 @@ func FingerScan(url string, method string) (string, string, string, string, []by
 	if len(cms) != 0 {
 		finger = strings.Join(cms, ",")
 	}
-	ServerValue := resp.Header["Server"]
-	retServerValue := ""
-	if len(ServerValue) != 0 {
-		retServerValue = ServerValue[0]
-	}
-	return finger, retServerValue, title, resp.Header.Get("Content-Type"), bodyBytes, resp.StatusCode
+
+	return finger, retServerValue, title, resp.Header.Get("Content-Type"), resp.Header.Get("Location"), []byte(bodyBytes), resp.StatusCode
 }
diff --git a/utils/port.go b/utils/port.go
@@ -146,7 +146,7 @@ func handleWorker(tasks <-chan ProtocolInfo, results chan ProtocolInfo, wg *sync
 func PortScan(IpRange string, PortRange string) {
 	ips, err := convertIPListToPool(strings.Split(IpRange, ","))
 	if err != nil {
-		Error("%s", err)
+		Info("%s", err)
 		return
 	}
 

diff --git a/utils/scan.go b/utils/scan.go
@@ -828,6 +828,7 @@ func (v *VScan) scanWithProbes(target Target, probes *[]Probe, config *Config) (
 						result.Timestamp = int32(time.Now().Unix())
 
 						found = true
+						Warning("Probe found=%t", found)
 						return result, nil
 					} else
 					// soft 匹配，记录结果