refactor: output filtering

hueristiq · Nov 20, 2023 · b60aafc · b60aafc
1 parent 395218d
commit b60aafc
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 66 deletions.
diff --git a/pkg/extractor/extractor.go b/pkg/extractor/extractor.go
@@ -11,7 +11,7 @@ func New(domain string) (extractor *regexp.Regexp, err error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
-	pattern := `(\w+[.])*` + regexp.QuoteMeta(domain) + `$`
+	pattern := `(\w+[.])*` + regexp.QuoteMeta(domain)
 
 	extractor, err = regexp.Compile(pattern)
 	if err != nil {

diff --git a/pkg/scraper/sources/certspotter/certspotter.go b/pkg/scraper/sources/certspotter/certspotter.go
@@ -4,9 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"regexp"
+	"strings"
 
-	"github.com/hueristiq/xsubfind3r/pkg/extractor"
 	"github.com/hueristiq/xsubfind3r/pkg/httpclient"
 	"github.com/hueristiq/xsubfind3r/pkg/scraper/sources"
 )
@@ -88,28 +87,13 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 			return
 		}
 
-		var regex *regexp.Regexp
-
-		regex, err = extractor.New(domain)
-		if err != nil {
-			result := sources.Result{
-				Type:   sources.Error,
-				Source: source.Name(),
-				Error:  err,
-			}
-
-			results <- result
-
-			return
-		}
-
 		for index := range getCTLogsSearchResData {
 			cert := getCTLogsSearchResData[index]
 
 			for index := range cert.DNSNames {
 				subdomain := cert.DNSNames[index]
 
-				if !regex.MatchString(subdomain) {
+				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 					continue
 				}
 
@@ -173,7 +157,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 				for index := range cert.DNSNames {
 					subdomain := cert.DNSNames[index]
 
-					if !regex.MatchString(subdomain) {
+					if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 						continue
 					}
 

diff --git a/pkg/scraper/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go
@@ -4,10 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"regexp"
 	"strings"
 
-	"github.com/hueristiq/xsubfind3r/pkg/extractor"
 	"github.com/hueristiq/xsubfind3r/pkg/httpclient"
 	"github.com/hueristiq/xsubfind3r/pkg/scraper/sources"
 )
@@ -64,29 +62,14 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		getNameValuesRes.Body.Close()
 
-		var regex *regexp.Regexp
-
-		regex, err = extractor.New(domain)
-		if err != nil {
-			result := sources.Result{
-				Type:   sources.Error,
-				Source: source.Name(),
-				Error:  err,
-			}
-
-			results <- result
-
-			return
-		}
-
 		for index := range getNameValuesResData {
 			record := getNameValuesResData[index]
 			subdomains := strings.Split(record.NameValue, "\n")
 
 			for index := range subdomains {
 				subdomain := subdomains[index]
 
-				if !regex.MatchString(subdomain) {
+				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 					continue
 				}
 

diff --git a/pkg/scraper/sources/otx/otx.go b/pkg/scraper/sources/otx/otx.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/hueristiq/xsubfind3r/pkg/httpclient"
 	"github.com/hueristiq/xsubfind3r/pkg/scraper/sources"
@@ -77,12 +78,16 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 		}
 
 		for index := range getPassiveDNSResData.PassiveDNS {
-			record := getPassiveDNSResData.PassiveDNS[index]
+			subdomain := getPassiveDNSResData.PassiveDNS[index].Hostname
+
+			if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
+				continue
+			}
 
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),
-				Value:  record.Hostname,
+				Value:  subdomain,
 			}
 
 			results <- result

diff --git a/pkg/scraper/sources/urlscan/urlscan.go b/pkg/scraper/sources/urlscan/urlscan.go
@@ -4,9 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"regexp"
+	"strings"
 
-	"github.com/hueristiq/xsubfind3r/pkg/extractor"
 	"github.com/hueristiq/xsubfind3r/pkg/httpclient"
 	"github.com/hueristiq/xsubfind3r/pkg/scraper/sources"
 )
@@ -125,36 +124,20 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 				break
 			}
 
-			var regex *regexp.Regexp
+			for index := range searchResData.Results {
+				subdomain := searchResData.Results[index].Page.Domain
+
+				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
+					continue
+				}
 
-			regex, err = extractor.New(domain)
-			if err != nil {
 				result := sources.Result{
-					Type:   sources.Error,
+					Type:   sources.Subdomain,
 					Source: source.Name(),
-					Error:  err,
+					Value:  subdomain,
 				}
 
 				results <- result
-
-				return
-			}
-
-			for index := range searchResData.Results {
-				result := searchResData.Results[index]
-				match := regex.FindAllString(result.Page.Domain, -1)
-
-				for index := range match {
-					subdomain := match[index]
-
-					result := sources.Result{
-						Type:   sources.Subdomain,
-						Source: source.Name(),
-						Value:  subdomain,
-					}
-
-					results <- result
-				}
 			}
 
 			if !searchResData.HasMore {