add custom headers support

README.md

@@ -33,16 +33,19 @@ Available Commands:
   weakpass    Start the application (Aliases: weak, wp, wk, ww)
 
 Flags:
-  -h, --help               help for godscan
-      --host string        singel host
-      --host-file string   host file
-  -v, --loglevel int       level of your log (default 2)
-  -o, --output string      output file to write log and results (default "result.log")
-      --private-ip         scan private ip
-      --proxy string       proxy
-      --ua string          set user agent (default "user agent")
-  -u, --url string         singel url
-      --url-file string    url file
+  -H, --headers stringArray   Custom headers
+  -h, --help                  help for godscan
+      --host string           singel host
+      --host-file string      host file
+  -v, --loglevel int          level of your log (default 2)
+  -o, --output string         output file to write log and results (default "result.log")
+      --private-ip            scan private ip
+      --proxy string          proxy
+      --ua string             set user agent (default "user agent")
+  -u, --url string            singel url
+      --url-file string       url file
+
+Use "godscan [command] --help" for more information about a command.
 ```
 
 
@@ -224,6 +227,7 @@ git push origin :refs/tags/v1.xx
 
 
 ## 更新说明
+* 2024-08-21 支持带cookie访问
 * 2024-07-01 新增全球端口top20000列表，`-p`参数保持不变
   * `--top 500`
   * `--top 500-1000`

cmd/root.go

@@ -13,7 +13,7 @@ import (
 	"github.com/spf13/viper"
 )
 
-var version = "v1.1.26"
+var version = "v1.1.27"
 
 func checkForUpdate(currentVersion string) {
 	ctx := context.Background()
@@ -55,6 +55,7 @@ type GlobalOptions struct {
 
 	DefaultUA     string
 	ScanPrivateIp bool
+	Headers       []string
 }
 
 var (
@@ -101,7 +102,7 @@ func SetProxyFromEnv() string {
 	return ""
 }
 
-func Execute() {
+func init() {
 	rootCmd.PersistentFlags().StringVarP(&GlobalOption.Url, "url", "u", "", "singel url")
 	rootCmd.PersistentFlags().StringVarP(&GlobalOption.UrlFile, "url-file", "", "", "url file")
 
@@ -117,6 +118,8 @@ func Execute() {
 
 	rootCmd.PersistentFlags().BoolVarP(&GlobalOption.ScanPrivateIp, "private-ip", "", false, "scan private ip")
 
+	rootCmd.PersistentFlags().StringArrayVarP(&GlobalOption.Headers, "headers", "H", nil, "Custom headers")
+
 	viper.BindPFlag("loglevel", rootCmd.PersistentFlags().Lookup("loglevel"))
 	viper.SetDefault("loglevel", 2)
 
@@ -129,13 +132,19 @@ func Execute() {
 	viper.BindPFlag("output", rootCmd.PersistentFlags().Lookup("output"))
 	viper.SetDefault("output", "result.log")
 
+	viper.BindPFlag("private-ip", rootCmd.PersistentFlags().Lookup("private-ip"))
+	viper.SetDefault("private-ip", false)
+
+	viper.BindPFlag("headers", rootCmd.PersistentFlags().Lookup("headers"))
+	viper.SetDefault("headers", []string{})
+
+}
+
+func Execute() {
 	if viper.GetString("proxy") != "" {
 		fmt.Fprintf(os.Stderr, "[%s] [%s] Proxy is %s\n", utils.GetCurrentTime(), color.New(color.FgCyan).Sprintf("%s", "INFO"), viper.GetString("proxy"))
 	} else {
 		fmt.Fprintf(os.Stderr, "[%s] [%s] Proxy is null\n", utils.GetCurrentTime(), color.New(color.FgCyan).Sprintf("%s", "INFO"))
 	}
-	viper.BindPFlag("private-ip", rootCmd.PersistentFlags().Lookup("private-ip"))
-	viper.SetDefault("private-ip", false)
-
 	cobra.CheckErr(rootCmd.Execute())
 }

utils/dirbrute.go

@@ -35,6 +35,9 @@ func DirBrute(baseUrl string, dir string) []string {
 		return []string{}
 	}
 	fullURL := baseURL.ResolveReference(&url.URL{Path: path.Join(baseURL.Path, dir)})
+	if strings.HasSuffix(dir, "/") && dir != "/" {
+		fullURL.Path += "/"
+	}
 	finger, _, title, contentType, location, respBody, statusCode := FingerScan(fullURL.String(), http.MethodGet, viper.GetBool("redirect"))
 	if statusCode == 200 || statusCode == 500 || statusCode == 302 {
 		result = CheckFinger(finger, title, fullURL.String(), contentType, location, respBody, statusCode)

utils/finger.go

@@ -53,7 +53,7 @@ func HttpGetServerHeader(Url string, NeedTitle bool, Method string) (string, str
 	if Method == http.MethodPost {
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 	}
-	req.Header.Set("User-Agent", viper.GetString("DefaultUA"))
+	SetHeaders(req)
 	resp, err := Client.Do(req)
 	if err != nil {
 		return "", "", "", err
@@ -129,9 +129,9 @@ func parseDir(fullPath string, MaxDepth int) []string {
 }
 
 func isValidUrl(Url string) bool {
-	arr := []string{"alicdn.com", "163.com", "nginx.com", "qq.com", "amap.com", "cnzz.com", "github.com", "apache.org", "gitlab.com", "centos.org"}
+	arr := []string{"alicdn.com", "163.com", "nginx.com", "qq.com", "amap.com", "cnzz.com", "github.com", "apache.org", "gitlab.com", "centos.org", "logout"}
 	for _, key := range arr {
-		if strings.Contains(Url, key) {
+		if strings.Contains(strings.ToLower(Url), key) {
 			return false
 		}
 	}
@@ -266,7 +266,7 @@ func Spider(RootPath string, Url string, depth int, filename string, myMap mapse
 	if err != nil {
 		return err
 	}
-	req.Header.Set("User-Agent", viper.GetString("DefaultUA"))
+	SetHeaders(req)
 	resp, err := Client.Do(req)
 	if err != nil {
 		return err
@@ -361,7 +361,7 @@ var icon_json string
 
 func IconDetect(Url string) (string, error) {
 	req, _ := http.NewRequest(http.MethodGet, Url, nil)
-	req.Header.Set("User-Agent", viper.GetString("DefaultUA"))
+	SetHeaders(req)
 	resp, err := Client.Do(req)
 
 	if err != nil {
@@ -396,7 +396,7 @@ func FindFaviconURL(urlStr string) (string, error) {
 
 	// 获取HTML内容
 	req, _ := http.NewRequest(http.MethodGet, urlStr, nil)
-	req.Header.Set("User-Agent", viper.GetString("DefaultUA"))
+	SetHeaders(req)
 	resp, err := Client.Do(req)
 	if err != nil {
 		return "", err

utils/matchrule.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/godspeedcurry/godscan/common"
-	"github.com/spf13/viper"
 	"golang.org/x/net/html/charset"
 )
 
@@ -144,8 +143,9 @@ func FingerScan(url string, method string, followRedirect bool) (string, string,
 		Fatal("%s %s xxx", url, err)
 		return common.NoFinger, "", "", "", "", nil, -1
 	}
-	req.Header.Set("User-Agent", viper.GetString("DefaultUA"))
 	req.Header.Set("Cookie", "rememberMe=me")
+	SetHeaders(req)
+
 	if !followRedirect {
 		Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse

utils/useful.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html"
 	"math/rand"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -18,6 +19,7 @@ import (
 	"github.com/godspeedcurry/godscan/common"
 	"github.com/mfonda/simhash"
 	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/viper"
 )
 
 const (
@@ -338,3 +340,21 @@ func WriteToCsv(filename string, data []string) {
 	}
 	writer.Flush()
 }
+
+func SetHeaders(req *http.Request) {
+	// 设置 User-Agent
+	req.Header.Set("User-Agent", viper.GetString("ua"))
+
+	// 设置自定义的请求头
+	headers := viper.GetStringSlice("headers")
+	for _, header := range headers {
+		parts := strings.SplitN(header, ":", 2)
+		if len(parts) != 2 {
+			Debug("Error: Invalid header format, correct format is 'Key: Value'")
+			continue
+		}
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+		req.Header.Set(key, value)
+	}
+}