From 055a0fb5fcc291b6f97310e51001a2b2130c7541 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:56:28 +0300 Subject: [PATCH] feat: Add BuiltWith as a source --- README.md | 5 +- internal/configuration/configuration.go | 15 +-- pkg/scraper/scraper.go | 3 + pkg/scraper/sources/builtwith/builtwith.go | 114 +++++++++++++++++++++ pkg/scraper/sources/sources.go | 16 +-- 5 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 pkg/scraper/sources/builtwith/builtwith.go diff --git a/README.md b/README.md index 6d268cb..2adff0d 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ## Post Installation -`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. +`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Chaos](https://chaos.projectdiscovery.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. Example `config.yaml`: @@ -113,6 +113,7 @@ sources: - alienvault - anubis - bevigil + - builtwith - chaos - commoncrawl - crtsh @@ -126,6 +127,8 @@ sources: keys: bevigil: - awA5nvpKU3N8ygkZ + builtwith: + - 7fcbaec4-dc49-472c-b837-3896cb255823 chaos: - d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39asdsd54bbc1aabb208c9acfb fullhunt: diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 4c2002e..9aea3c6 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -87,13 +87,14 @@ func CreateUpdate(path string) (err error) { Version: VERSION, Sources: sources.List, Keys: sources.Keys{ - Bevigil: []string{}, - Chaos: []string{}, - Fullhunt: []string{}, - GitHub: []string{}, - Intelx: []string{}, - Shodan: []string{}, - URLScan: []string{}, + Bevigil: []string{}, + BuiltWith: []string{}, + Chaos: []string{}, + Fullhunt: []string{}, + GitHub: []string{}, + Intelx: []string{}, + Shodan: []string{}, + URLScan: []string{}, }, } diff --git a/pkg/scraper/scraper.go b/pkg/scraper/scraper.go index 341c022..5d5aab0 100644 --- a/pkg/scraper/scraper.go +++ b/pkg/scraper/scraper.go @@ -7,6 +7,7 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/anubis" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/bevigil" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/builtwith" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/chaos" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/commoncrawl" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources/crtsh" @@ -85,6 +86,8 @@ func New(options *Options) (scraper *Scraper) { scraper.Sources[source] = &anubis.Source{} case "bevigil": scraper.Sources[source] = &bevigil.Source{} + case "builtwith": + scraper.Sources[source] = &builtwith.Source{} case "chaos": scraper.Sources[source] = &chaos.Source{} case "commoncrawl": diff --git a/pkg/scraper/sources/builtwith/builtwith.go b/pkg/scraper/sources/builtwith/builtwith.go new file mode 100644 index 0000000..c75aa34 --- /dev/null +++ b/pkg/scraper/sources/builtwith/builtwith.go @@ -0,0 +1,114 @@ +package builtwith + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/hueristiq/xsubfind3r/pkg/httpclient" + "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" +) + +type getDomainInfoResponse struct { + Results []struct { + Result struct { + Paths []struct { + Domain string `json:"Domain"` + URL string `json:"Url"` + SubDomain string `json:"SubDomain"` + } `json:"Paths"` + } `json:"Result"` + } `json:"Results"` +} + +type Source struct{} + +func (source *Source) Run(config *sources.Configuration, domain string) <-chan sources.Result { + results := make(chan sources.Result) + + go func() { + defer close(results) + + var err error + + var key string + + key, err = sources.PickRandom(config.Keys.BuiltWith) + if key == "" || err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + return + } + + getDomainInfoReqURL := fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", key, domain) + + var getDomainInfoRes *http.Response + + getDomainInfoRes, err = httpclient.SimpleGet(getDomainInfoReqURL) + if err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getDomainInfoRes.Body.Close() + + return + } + + var getDomainInfoResData getDomainInfoResponse + + if err = json.NewDecoder(getDomainInfoRes.Body).Decode(&getDomainInfoResData); err != nil { + result := sources.Result{ + Type: sources.Error, + Source: source.Name(), + Error: err, + } + + results <- result + + getDomainInfoRes.Body.Close() + + return + } + + getDomainInfoRes.Body.Close() + + for index := range getDomainInfoResData.Results { + item := getDomainInfoResData.Results[index] + + for index := range item.Result.Paths { + path := item.Result.Paths[index] + + value := path.Domain + + if path.SubDomain != "" { + value = path.SubDomain + "." + value + } + + result := sources.Result{ + Type: sources.Subdomain, + Source: source.Name(), + Value: value, + } + + results <- result + } + } + }() + + return results +} + +func (source *Source) Name() string { + return "builtwith" +} diff --git a/pkg/scraper/sources/sources.go b/pkg/scraper/sources/sources.go index c1ed2c2..f511b02 100644 --- a/pkg/scraper/sources/sources.go +++ b/pkg/scraper/sources/sources.go @@ -14,13 +14,14 @@ type Configuration struct { } type Keys struct { - Bevigil []string `yaml:"bevigil"` - Chaos []string `yaml:"chaos"` - Fullhunt []string `yaml:"fullhunt"` - GitHub []string `yaml:"github"` - Intelx []string `yaml:"intelx"` - Shodan []string `yaml:"shodan"` - URLScan []string `yaml:"urlscan"` + Bevigil []string `yaml:"bevigil"` + BuiltWith []string `yaml:"builtwith"` + Chaos []string `yaml:"chaos"` + Fullhunt []string `yaml:"fullhunt"` + GitHub []string `yaml:"github"` + Intelx []string `yaml:"intelx"` + Shodan []string `yaml:"shodan"` + URLScan []string `yaml:"urlscan"` } // Result is a result structure returned by a source. @@ -43,6 +44,7 @@ const ( var List = []string{ "anubis", "bevigil", + "builtwith", "chaos", "commoncrawl", "crtsh",