[feat] 优化泛解析

CHANGELOG.md

@@ -1,7 +1,10 @@
 # Changelog
 
+## v0.0.8
+- [feat] 优化泛解析，添加参数 mI, 爆破时如果超出一定数量的域名指向同一个 ip，则认为是泛解析(默认 100)
+
 ## v0.0.7
-- [fix] 去除静默模式下 banner 输出
+- [fix] 修复静默模式下还会输出 banner 的 bug
 
 ## v0.0.6
 - [fix] 修复-s 指定源不生效的 bug

README.md

@@ -145,6 +145,7 @@ func main() {
 		Brute:				true,
 		Verify:             true,	// 验证找到的域名 
 		RemoveWildcard: 	true,	// 泛解析过滤 
+		MaxIPs:             100,    // 爆破时如果超出一定数量的域名指向同一个 ip，则认为是泛解析
 		Silent: 			false,	// 是否为静默模式，只输出找到的域名
 		DNS: 				"cn",	// dns 服务器区域选择，根据目标选择不同区域得到的结果不同，国内网站的话，选择 cn，dns 爆破结果比较多
 		BruteWordlist:      "",		// 爆破子域的域名字典，不填则使用内置的

pkg/active/active.go

@@ -7,7 +7,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 )
 
-func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fileName string, level int, levelDict string, resolvers []string, wildcardIPs map[string]struct{}) map[string]resolve.HostEntry {
+func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fileName string, level int, levelDict string, resolvers []string, wildcardIPs map[string]struct{}, maxIPs int) (map[string]resolve.HostEntry, map[string]struct{}) {
 	gologger.Info().Msgf("Start DNS blasting of %s", domain)
 	var levelDomains []string
 	if levelDict != "" {
@@ -28,6 +28,7 @@ func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fi
 		Output:       "",
 		Silent:       silent,
 		WildcardIPs:  wildcardIPs,
+		MaxIPs:		  maxIPs,
 		TimeOut:      5,
 		Retry:        6,
 		Level:        level,  				// 枚举几级域名，默认为2，二级域名,
@@ -43,13 +44,13 @@ func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fi
 		gologger.Fatal().Msgf("%s", err)
 	}
 
-	enumMap := r.RunEnumeration(uniqueMap, ctx)
+	enumMap, wildcardIPs := r.RunEnumeration(uniqueMap, ctx)
 
 	r.Close()
-	return enumMap
+	return enumMap, wildcardIPs
 }
 
-func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, resolvers []string, wildcardIPs map[string]struct{}) map[string]resolve.HostEntry {
+func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, resolvers []string, wildcardIPs map[string]struct{}, maxIPs int) (map[string]resolve.HostEntry, map[string]struct{}) {
 	gologger.Info().Msgf("Start to verify the collected sub domain name results, a total of %d", len(uniqueMap))
 
 	opt := &Options {
@@ -60,6 +61,7 @@ func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, resolvers []str
 		Output:       "",
 		Silent:       silent,
 		WildcardIPs:  wildcardIPs,
+		MaxIPs:		  maxIPs,
 		TimeOut:      5,
 		Retry:        6,
 		Method:       "verify",
@@ -71,9 +73,9 @@ func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, resolvers []str
 		gologger.Fatal().Msgf("%s", err)
 	}
 
-	AuniqueMap := r.RunEnumerationVerify(uniqueMap, ctx)
+	AuniqueMap, wildcardIPs := r.RunEnumerationVerify(uniqueMap, ctx)
 
 	r.Close()
 
-	return AuniqueMap
+	return AuniqueMap, wildcardIPs
 }

pkg/active/options.go

@@ -14,6 +14,8 @@ type Options struct {
 	Output       string // 输出文件名
 	Silent       bool
 	WildcardIPs  map[string]struct{}
+	WildcardIPsAc  map[string]struct{}
+	MaxIPs       int
 	TimeOut      int
 	Retry        int
 	Method       string // verify模式 enum模式 test模式

pkg/active/recv.go

@@ -7,7 +7,6 @@ import (
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
-	"github.com/projectdiscovery/gologger"
 	"sync/atomic"
 	"time"
 )
@@ -108,7 +107,7 @@ func (r *runner) recvChanel(ctx context.Context) error {
 			if !skip {
 				select {
 				case <-ctx.Done():
-					gologger.Error().Msg("recvChanel ctx.Done()............")
+					return nil
 				default:
 					atomic.AddUint64(&r.successIndex, 1)
 					r.recver <- RecvResult {

pkg/active/runner.go

@@ -140,7 +140,7 @@ func (r *runner) PrintStatus() {
 	gologger.Info().Msgf("\rSuccess:%d Send:%d Queue:%d Accept:%d Fail:%d Elapsed:%ds", r.successIndex, r.sendIndex, queue, r.recvIndex, r.faildIndex, tc)
 }
 
-func (r *runner) RunEnumeration(uniqueMap map[string]resolve.HostEntry, ctx context.Context) map[string]resolve.HostEntry{
+func (r *runner) RunEnumeration(uniqueMap map[string]resolve.HostEntry, ctx context.Context) (map[string]resolve.HostEntry, map[string]struct{}) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -160,6 +160,8 @@ func (r *runner) RunEnumeration(uniqueMap map[string]resolve.HostEntry, ctx cont
 	}
 
 	go func(ctx context.Context) {
+		ipsMap := make(map[string]int)
+
 		for result := range r.recver {
 			var cnames []string
 			var ips []string
@@ -180,6 +182,20 @@ func (r *runner) RunEnumeration(uniqueMap map[string]resolve.HostEntry, ctx cont
 					continue
 				}
 
+				/*
+				todo 这里只是记录了第一个 ip , 还是应该记录所有解析出的 ip ？
+				比如 这个域名 spotifyforbrands.com， 存在泛解析，也只是解析出一个 ip
+				*/
+
+				// ip 都记录一下 ，超过阈值，则认为是泛解析 ip
+				ipsMap[ips[0]] +=1
+
+				// 记录这个 ip 到泛解析 ip 列表中， 最终在返回结果中去除
+				if ipsMap[ips[0]] > r.options.MaxIPs {
+					r.options.WildcardIPsAc[ips[0]] = struct{}{}
+					continue
+				}
+
 				var ipPorts map[string][]int
 
 				if uniqueMap[result.Subdomain].IpPorts != nil {
@@ -195,6 +211,7 @@ func (r *runner) RunEnumeration(uniqueMap map[string]resolve.HostEntry, ctx cont
 						skip = true
 						break
 					}
+
 					if ipPorts[ip] == nil {
 						ipPorts[ip] = nil
 					}
@@ -226,20 +243,20 @@ func (r *runner) RunEnumeration(uniqueMap map[string]resolve.HostEntry, ctx cont
 			r.PrintStatus()
 			if isLoadOver {
 				if r.hm.Length() <= 0 {
-					return uniqueMap
+					return uniqueMap, r.options.WildcardIPsAc
 				}
 			}
 		case <-r.fisrtloadChanel:
 			go r.retry(ctx) // 遍历hm，依次重试
 			isLoadOver = true
 		case <-ctx.Done():
-			return uniqueMap
+			return uniqueMap, r.options.WildcardIPsAc
 		}
 	}
 
 }
 
-func (r *runner) RunEnumerationVerify(uniqueMap map[string]resolve.HostEntry, ctx context.Context) map[string]resolve.HostEntry{
+func (r *runner) RunEnumerationVerify(uniqueMap map[string]resolve.HostEntry, ctx context.Context) (map[string]resolve.HostEntry, map[string]struct{}) {
 	AuniqueMap := make(map[string]resolve.HostEntry)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -256,6 +273,7 @@ func (r *runner) RunEnumerationVerify(uniqueMap map[string]resolve.HostEntry, ct
 	}
 
 	go func(ctx context.Context) {
+		ipsMap := make(map[string]int)
 		for result := range r.recver {
 			var cnames []string
 			var ips []string
@@ -273,6 +291,15 @@ func (r *runner) RunEnumerationVerify(uniqueMap map[string]resolve.HostEntry, ct
 				continue
 			}
 
+			// ip 都记录一下 ，超过阈值，则认为是泛解析 ip
+			ipsMap[ips[0]] +=1
+
+			// 记录这个 ip 到泛解析 ip 列表中， 最终在返回结果中去除
+			if ipsMap[ips[0]] > r.options.MaxIPs {
+				r.options.WildcardIPsAc[ips[0]] = struct{}{}
+				continue
+			}
+
 			var ipPorts map[string][]int
 
 			if uniqueMap[result.Subdomain].IpPorts != nil {
@@ -320,7 +347,7 @@ func (r *runner) RunEnumerationVerify(uniqueMap map[string]resolve.HostEntry, ct
 			r.PrintStatus()
 			if isLoadOver {
 				if r.hm.Length() <= 0 {
-					return AuniqueMap
+					return AuniqueMap, r.options.WildcardIPsAc
 				}
 			}
 		case <-r.fisrtloadChanel:

pkg/runner/banner.go

@@ -17,7 +17,7 @@ const banner = `
 `
 
 // Version is the current version of Starmap
-const Version = `v0.0.7`
+const Version = `v0.0.8`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

pkg/runner/enumerate.go

@@ -131,6 +131,8 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 
 	var wildcardIPs map[string]struct{}
 
+	var wildcardIPsAc map[string]struct{}
+
 	if r.options.RemoveWildcard {
 		gologger.Info().Msgf("%s 检测泛解析", domain)
 		var err error
@@ -150,34 +152,35 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 		}
 
 		if len(wildcardIPs) > 0 {
-			gologger.Info().Msgf("域名:%s 存在泛解析, 自动过滤泛解析\n", domain)
+			gologger.Info().Msgf("域名:%s 存在泛解析, 自动过滤泛解析， %v\n", domain, wildcardIPs)
 		}
 	}
 
-
 	if r.options.Verify { // 验证模式
 		l := len(uniqueMap)
-		uniqueMap = active.Verify(uniqueMap, r.options.Silent, r.Resolvers, wildcardIPs)
+		uniqueMap, wildcardIPsAc = active.Verify(uniqueMap, r.options.Silent, r.Resolvers, wildcardIPs, r.options.MaxIps)
 		gologger.Info().Msgf("A total of %d were collected in passive mode, and %d were verified to be alive", l, len(uniqueMap))
 
 	} else {
 		gologger.Info().Msgf("Passive acquisition end, Found %d subdomains.", len(uniqueMap))
 	}
 
+	time.Sleep(5*time.Second)
 	if r.options.Brute {
 		if r.options.Number > 1 {
 			n := make(map[string]resolve.HostEntry)
 			// dns 爆破次数
 			for i := 1; i <= r.options.Number; i++ {
-				test := active.Enum(domain, uniqueMap, r.options.Silent, r.options.BruteWordlist, r.options.Level, r.options.LevelDic, r.Resolvers, wildcardIPs)
+				var test map[string]resolve.HostEntry
+				test, wildcardIPsAc = active.Enum(domain, uniqueMap, r.options.Silent, r.options.BruteWordlist, r.options.Level, r.options.LevelDic, r.Resolvers, wildcardIPs, r.options.MaxIps)
 				if i > 1 {
 					n = util.MergeMap(uniqueMap, test)
 				}
 				uniqueMap = test
 			}
 			uniqueMap = n
 		} else {
-			uniqueMap = active.Enum(domain, uniqueMap, r.options.Silent, r.options.BruteWordlist, r.options.Level, r.options.LevelDic, r.Resolvers, wildcardIPs)
+			uniqueMap, wildcardIPsAc = active.Enum(domain, uniqueMap, r.options.Silent, r.options.BruteWordlist, r.options.Level, r.options.LevelDic, r.Resolvers, wildcardIPs, r.options.MaxIps)
 		}
 
 	}
@@ -188,6 +191,25 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 		uniqueMap = subTakeOver.Process(uniqueMap, r.options.SAll, r.options.Verbose)
 	}
 
+
+
+	// 泛解析再次处理
+	if len(wildcardIPsAc) > 0 {
+		for _, result := range uniqueMap {
+			if result.IpPorts != nil {
+				var ips []string
+				for k, _ := range result.IpPorts {
+					ips = append(ips, k)
+				}
+
+				if _, ok := wildcardIPsAc[ips[0]]; ok {
+					delete(uniqueMap, result.Host)
+				}
+			}
+		}
+	}
+
+
 	outputter := NewOutputter(r.options.JSON)
 
 	// Now output all results in output writers

pkg/runner/options.go

@@ -62,6 +62,7 @@ type Options struct {
 	Takeover		   bool   // subdomain takeover
 	SAll               bool   // Request to test each URL (by default, only the URL matching CNAME is requested to test).
 	MaxWildcardChecks  int		// MaxWildcardChecks Number of random domain names
+	MaxIps             int
 }
 
 
@@ -145,6 +146,8 @@ func ParseOptions() *Options {
 		flagSet.StringVar(&options.DNS, "dns", "cn", "DNS server, cn:China dns, in:International, all:(cn+in DNS), conf:(read ./config/Starmap/config.yaml), Select according to the target. \nDNS服务器，默认国内的服务器(cn)(cn: 表示使用国内的 dns, in:国外 dns，all: 全部内置 dns, conf: 从配置文件 ./config/Starmap/config.yaml获取)，根据目标选择"),
 		flagSet.BoolVarP(&options.RemoveWildcard, "active", "rW", false, "Domain name pan resolution filtering\n爆破时过滤泛解析(default false)"),
 		flagSet.IntVar(&options.MaxWildcardChecks, "mW", 0, "Number of random domain names during universal resolution detection(default len(resolvers)*2)\n泛解析检测时的随机域名数量(default len(resolvers)*2)"),
+		flagSet.IntVar(&options.MaxIps, "mI", 100, "When blasting, if more than a certain number of domain names point to the same IP, it is considered as universal resolution(default 100)\n爆破时如果超出一定数量的域名指向同一个 ip，则认为是泛解析(default 100)"),
+
 	)
 
 	createGroup(flagSet, "takeover", "subdomain takeover",