Merge pull request #4 from hueristiq/handle-multiple-domains

Handle multiple domains
hueristiq · Jul 22, 2023 · b3639f1 · b3639f1
2 parents 6eeda8d + fd3fac7
commit b3639f1
Show file tree

Hide file tree

Showing 19 changed files with 253 additions and 136 deletions.
diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go
@@ -8,45 +8,56 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/hueristiq/hqgolog"
 	"github.com/hueristiq/hqgolog/formatter"
 	"github.com/hueristiq/hqgolog/levels"
 	"github.com/hueristiq/xsubfind3r/internal/configuration"
 	"github.com/hueristiq/xsubfind3r/pkg/xsubfind3r"
+	"github.com/hueristiq/xsubfind3r/pkg/xsubfind3r/sources"
 	"github.com/logrusorgru/aurora/v3"
 	"github.com/spf13/pflag"
 )
 
 var (
 	au aurora.Aurora
 
-	domain string
+	domainsSlice        []string
+	domainsListFilePath string
 
-	sourcesToExclude []string
 	listSources      bool
 	sourcesToUse     []string
+	sourcesToExclude []string
 
-	monochrome bool
-	output     string
-	verbosity  string
+	threads int
+
+	monochrome      bool
+	output          string
+	outputDirectory string
+	verbosity       string
 
 	YAMLConfigFile string
 )
 
 func init() {
 	// defaults
+	defaultThreads := 50
 	defaultYAMLConfigFile := "~/.hueristiq/xsubfind3r/config.yaml"
 
 	// Handle CLI arguments, flags & help message (pflag)
-	pflag.StringVarP(&domain, "domain", "d", "", "")
+	pflag.StringSliceVarP(&domainsSlice, "domain", "d", []string{}, "")
+	pflag.StringVarP(&domainsListFilePath, "list", "l", "", "")
 
-	pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "")
-	pflag.BoolVarP(&listSources, "sources", "s", false, "")
+	pflag.BoolVar(&listSources, "sources", false, "")
 	pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "")
+	pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "")
+
+	pflag.IntVarP(&threads, "threads", "t", defaultThreads, "")
 
 	pflag.BoolVar(&monochrome, "no-color", false, "")
 	pflag.StringVarP(&output, "output", "o", "", "")
+	pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "")
 	pflag.StringVarP(&verbosity, "verbosity", "v", string(levels.LevelInfo), "")
 
 	pflag.StringVarP(&YAMLConfigFile, "configuration", "c", defaultYAMLConfigFile, "")
@@ -58,21 +69,26 @@ func init() {
 		h := "USAGE:\n"
 		h += "  xsubfind3r [OPTIONS]\n"
 
-		h += "\nTARGET:\n"
-		h += " -d, --domain string              target domain\n"
+		h += "\nINPUT:\n"
+		h += " -d, --domain string[]                 target domains\n"
+		h += " -l, --list string                     target domains' list file path\n"
 
 		h += "\nSOURCES:\n"
-		h += " -e,  --exclude-sources string    sources to exclude\n"
-		h += " -s,  --sources bool              list sources\n"
-		h += " -u,  --use-sources string        sources to use\n"
+		h += "      --sources bool                   list supported sources\n"
+		h += " -u,  --sources-to-use string[]        comma(,) separeted sources to use\n"
+		h += " -e,  --sources-to-exclude string[]    comma(,) separeted sources to exclude\n"
+
+		h += "\nOPTIMIZATION:\n"
+		h += fmt.Sprintf(" -t,  --threads int                    number of threads (default: %d)\n", defaultThreads)
 
 		h += "\nOUTPUT:\n"
-		h += "     --no-color bool              no colored mode\n"
-		h += " -o, --output string              output subdomains file path\n"
-		h += fmt.Sprintf(" -v, --verbosity string           debug, info, warning, error, fatal or silent (default: %s)\n", string(levels.LevelInfo))
+		h += "     --no-color bool                   disable colored output\n"
+		h += " -o, --output string                   output subdomains' file path\n"
+		h += " -O, --output-directory string         output subdomains' directory path\n"
+		h += fmt.Sprintf(" -v, --verbosity string                debug, info, warning, error, fatal or silent (default: %s)\n", string(levels.LevelInfo))
 
 		h += "\nCONFIGURATION:\n"
-		h += fmt.Sprintf(" -c,  --configuration string      configuration file path (default: %s)\n", defaultYAMLConfigFile)
+		h += fmt.Sprintf(" -c,  --configuration string           configuration file path (default: %s)\n", defaultYAMLConfigFile)
 
 		fmt.Fprintln(os.Stderr, h)
 	}
@@ -103,21 +119,21 @@ func init() {
 }
 
 func main() {
-	// Print Banner
+	// Print banner.
 	if verbosity != string(levels.LevelSilent) {
 		fmt.Fprintln(os.Stderr, configuration.BANNER)
 	}
 
-	// Read in configuration
+	// Read in configuration.
 	config, err := configuration.Read(YAMLConfigFile)
 	if err != nil {
 		hqgolog.Fatal().Msg(err.Error())
 	}
 
-	// List suported sources
+	// List suported sources.
 	if listSources {
-		hqgolog.Info().Msgf("listing %v current supported sources", au.Underline(strconv.Itoa(len(config.Sources))).Bold())
-		hqgolog.Info().Msgf("sources with %v needs a key or token", au.Underline("*").Bold())
+		hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(config.Sources))).Bold())
+		hqgolog.Info().Msgf("sources marked with %v need key(s) or token(s) to work.", au.Underline("*").Bold())
 		hqgolog.Print().Msg("")
 
 		needsKey := make(map[string]interface{})
@@ -140,63 +156,153 @@ func main() {
 		os.Exit(0)
 	}
 
-	// Find subdomains
-	if verbosity != string(levels.LevelSilent) {
-		hqgolog.Info().Msgf("finding subdomains for %v.", au.Underline(domain).Bold())
-		hqgolog.Print().Msg("")
-	}
+	domains := make(chan string, threads)
 
-	options := &xsubfind3r.Options{
-		Domain:           domain,
-		SourcesToExclude: sourcesToExclude,
-		SourcesToUSe:     sourcesToUse,
-		Keys:             config.Keys,
-	}
+	// Load input domains
+	go func() {
+		defer close(domains)
 
-	finder := xsubfind3r.New(options)
-	subdomains := finder.Find()
+		// input domains: slice
+		for _, domain := range domainsSlice {
+			domains <- domain
+		}
 
-	if output != "" {
-		// Create output file path directory
-		directory := filepath.Dir(output)
+		// input domains: file
+		if domainsListFilePath != "" {
+			file, err := os.Open(domainsListFilePath)
+			if err != nil {
+				hqgolog.Error().Msg(err.Error())
+			}
 
-		if _, err := os.Stat(directory); os.IsNotExist(err) {
-			if err = os.MkdirAll(directory, os.ModePerm); err != nil {
-				hqgolog.Fatal().Msg(err.Error())
+			scanner := bufio.NewScanner(file)
+
+			for scanner.Scan() {
+				domain := scanner.Text()
+
+				if domain != "" {
+					domains <- domain
+				}
+			}
+
+			if err := scanner.Err(); err != nil {
+				hqgolog.Error().Msg(err.Error())
 			}
 		}
 
-		// Create output file
-		file, err := os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		// input domains: stdin
+		if hasStdin() {
+			scanner := bufio.NewScanner(os.Stdin)
+
+			for scanner.Scan() {
+				domain := scanner.Text()
+
+				if domain != "" {
+					domains <- domain
+				}
+			}
+
+			if err := scanner.Err(); err != nil {
+				hqgolog.Error().Msg(err.Error())
+			}
+		}
+	}()
+
+	// Find and output subdomains.
+	var consolidatedWriter *bufio.Writer
+
+	if output != "" {
+		directory := filepath.Dir(output)
+
+		mkdir(directory)
+
+		consolidatedFile, err := os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			hqgolog.Fatal().Msg(err.Error())
 		}
 
-		defer file.Close()
+		defer consolidatedFile.Close()
 
-		// Write subdomains output file and print on screen
-		writer := bufio.NewWriter(file)
+		consolidatedWriter = bufio.NewWriter(consolidatedFile)
+	}
 
-		for subdomains := range subdomains {
-			if verbosity == string(levels.LevelSilent) {
-				hqgolog.Print().Msg(subdomains.Value)
-			} else {
-				hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomains.Source), subdomains.Value)
+	if outputDirectory != "" {
+		mkdir(outputDirectory)
+	}
+
+	wg := &sync.WaitGroup{}
+
+	for i := 0; i < threads; i++ {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+
+			options := &xsubfind3r.Options{
+				SourcesToExclude: sourcesToExclude,
+				SourcesToUSe:     sourcesToUse,
+				Keys:             config.Keys,
 			}
 
-			fmt.Fprintln(writer, subdomains.Value)
-		}
+			finder := xsubfind3r.New(options)
+
+			for domain := range domains {
+				subdomains := finder.Find(domain)
+
+				switch {
+				case output != "":
+					processSubdomains(consolidatedWriter, subdomains, verbosity)
+				case outputDirectory != "":
+					domainFile, err := os.OpenFile(filepath.Join(outputDirectory, domain+".txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+					if err != nil {
+						hqgolog.Fatal().Msg(err.Error())
+					}
+
+					domainWriter := bufio.NewWriter(domainFile)
 
-		if err = writer.Flush(); err != nil {
+					processSubdomains(domainWriter, subdomains, verbosity)
+				default:
+					processSubdomains(nil, subdomains, verbosity)
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+}
+
+func hasStdin() bool {
+	stat, err := os.Stdin.Stat()
+	if err != nil {
+		return false
+	}
+
+	isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0
+	isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0
+
+	return isPipedFromChrDev || isPipedFromFIFO
+}
+
+func mkdir(path string) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		if err = os.MkdirAll(path, os.ModePerm); err != nil {
 			hqgolog.Fatal().Msg(err.Error())
 		}
-	} else {
-		// Print subdomains on screen
-		for subdomains := range subdomains {
-			if verbosity == string(levels.LevelSilent) {
-				hqgolog.Print().Msg(subdomains.Value)
-			} else {
-				hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomains.Source), subdomains.Value)
+	}
+}
+
+func processSubdomains(writer *bufio.Writer, subdomains chan sources.Subdomain, verbosity string) {
+	for subdomain := range subdomains {
+		if verbosity == string(levels.LevelSilent) {
+			hqgolog.Print().Msg(subdomain.Value)
+		} else {
+			hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(subdomain.Source), subdomain.Value)
+		}
+
+		if writer != nil {
+			fmt.Fprintln(writer, subdomain.Value)
+
+			if err := writer.Flush(); err != nil {
+				hqgolog.Fatal().Msg(err.Error())
 			}
 		}
 	}

diff --git a/go.mod b/go.mod
@@ -9,13 +9,13 @@ require (
 	github.com/logrusorgru/aurora/v3 v3.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
-	github.com/valyala/fasthttp v1.38.0
+	github.com/valyala/fasthttp v1.48.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/andybalholm/brotli v1.0.4 // indirect
-	github.com/klauspost/compress v1.15.0 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/net v0.11.0 // indirect
 	golang.org/x/sys v0.9.0 // indirect

diff --git a/go.sum b/go.sum
@@ -1,13 +1,13 @@
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
-github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg=
 github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47 h1:LSeeeVmzUmykvyAS/liWZ+yQuMjM/1462m9s+WWV9YI=
 github.com/hueristiq/hqgourl v0.0.0-20230623114406-412908c09f47/go.mod h1:8NAT2ECb69qzGf2d/ty0PVE3M3HK/+fXLtri2c47wQE=
-github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
-github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
+github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4=
 github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -16,28 +16,14 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.38.0 h1:yTjSSNjuDi2PPvXY2836bIwLmiTS2T4T9p1coQshpco=
-github.com/valyala/fasthttp v1.38.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
-github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
+github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
 golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
 golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=