diff --git a/README.md b/README.md index 12457a7040..32ac65211f 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,6 @@ CONFIGURATIONS: -rsr, -response-size-read int max response size to read in bytes (default 10485760) -rss, -response-size-save int max response size to read in bytes (default 1048576) -reset reset removes all nuclei configuration and data files (including nuclei-templates) - -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me) diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index c822480edd..c580d43c4a 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -23,8 +23,8 @@ builds: flags: - -trimpath -- main: cmd/tmc/main.go - binary: tmc +- main: cmd/cve-annotate/main.go + binary: cve-annotate id: annotate env: diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go new file mode 100644 index 0000000000..127cd5c62d --- /dev/null +++ b/v2/cmd/cve-annotate/main.go @@ -0,0 +1,650 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/nvd" + "github.com/projectdiscovery/retryablehttp-go" + sliceutil "github.com/projectdiscovery/utils/slice" + stringsutil "github.com/projectdiscovery/utils/strings" + "gopkg.in/yaml.v3" +) + +const ( + yamlIndentSpaces = 2 +) + +var cisaKnownExploitedVulnerabilities map[string]struct{} + +// allTagsRegex is a list of all tags in nuclei templates except id, info, and - +var allTagsRegex []*regexp.Regexp +var defaultOpts = types.DefaultOptions() + +func init() { + var tm templates.Template + t := reflect.TypeOf(tm) + for i := 0; i < t.NumField(); i++ { + tag := t.Field(i).Tag.Get("yaml") + if strings.Contains(tag, ",") { + tag = strings.Split(tag, ",")[0] + } + // ignore these tags + if tag == "id" || tag == "info" || tag == "" || tag == "-" { + continue + } + re := regexp.MustCompile(tag + `:\s*\n`) + allTagsRegex = append(allTagsRegex, re) + } + + defaultOpts := types.DefaultOptions() + // need to set headless to true for headless templates + defaultOpts.Headless = true + if err := protocolstate.Init(defaultOpts); err != nil { + gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) + } + if err := protocolinit.Init(defaultOpts); err != nil { + gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) + } + if err := fetchCISAKnownExploitedVulnerabilities(); err != nil { + panic(err) + } +} + +var ( + input = flag.String("i", "", "Templates to annotate") + verbose = flag.Bool("v", false, "show verbose output") +) + +func main() { + flag.Parse() + + if *input == "" { + log.Fatalf("invalid input, see -h\n") + } + if strings.HasPrefix(*input, "~/") { + home, err := os.UserHomeDir() + if err != nil { + log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err) + } + *input = filepath.Join(home, (*input)[2:]) + } + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + if *verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + } + if err := process(); err != nil { + gologger.Error().Msgf("could not process: %s\n", err) + } +} + +func process() error { + tempDir, err := os.MkdirTemp("", "nuclei-nvd-%s") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + client := nvd.NewClientV2() + templateCatalog := disk.NewCatalog(filepath.Dir(*input)) + paths, err := templateCatalog.GetTemplatePath(*input) + if err != nil { + return err + } + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return err + } + dataString := string(data) + // try to fill max-requests + dataString, err = parseAndAddMaxRequests(templateCatalog, path, dataString) + if err != nil { + gologger.Error().Msgf("Could not compile max request %s: %s\n", path, err) + } + // try to resolve references to tags + dataString, err = parseAndAddReferenceBasedTags(path, dataString) + if err != nil { + gologger.Error().Msgf("Could not parse reference tags %s: %s\n", path, err) + continue + } + // try and fill CVE data + getCVEData(client, path, dataString) + } + return nil +} + +var ( + idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)") + severityRegex = regexp.MustCompile(`severity: ([a-z]+)`) +) + +const maxReferenceCount = 5 + +// dead sites to skip for references +var badRefs = []string{ + "osvdb.org/", + "securityfocus.com/", + "archives.neohapsis.com/", + "iss.net/", + "ntelbras.com/", + "andmp.com/", + "blacklanternsecurity.com/", + "pwnwiki.org/", + "0dayhack.net/", + "correkt.horse/", + "poc.wgpsec.org/", + "ctf-writeup.revers3c.com/", + "secunia.com/", +} + +func getCVEData(client *nvd.ClientV2, filePath, data string) { + matches := idRegex.FindAllStringSubmatch(data, 1) + if len(matches) == 0 { + return + } + cveName := matches[0][1] + + // Perform CISA Known-exploited-vulnerabilities tag annotation + // if we discover it has been exploited. + var err error + if cisaKnownExploitedVulnerabilities != nil { + _, ok := cisaKnownExploitedVulnerabilities[strings.ToLower(cveName)] + if ok { + data, err = parseAndAddCISAKevTagTemplate(filePath, data) + } + } + if err != nil { + gologger.Error().Msgf("Could not parse cisa data %s: %s\n", cveName, err) + return + } + + severityMatches := severityRegex.FindAllStringSubmatch(data, 1) + if len(severityMatches) == 0 { + return + } + severityValue := severityMatches[0][1] + + cveItem, err := client.FetchCVE(cveName) + if err != nil { + gologger.Error().Msgf("Could not fetch cve %s: %s\n", cveName, err) + return + } + var cweID []string + for _, weaknessData := range cveItem.Cve.Weaknesses { + for _, description := range weaknessData.Description { + cweID = append(cweID, description.Value) + } + } + cvssData, err := getPrimaryCVSSData(cveItem) + if err != nil { + gologger.Error().Msgf("Could not get CVSS data %s: %s\n", cveName, err) + return + } + cvssScore := cvssData.BaseScore + cvssMetrics := cvssData.VectorString + + // Perform some hacky string replacement to place the metadata in templates + infoBlockIndexData := data[strings.Index(data, "info:"):] + requestsIndex := strings.Index(infoBlockIndexData, "requests:") + networkIndex := strings.Index(infoBlockIndexData, "network:") + variablesIndex := strings.Index(infoBlockIndexData, "variables:") + if requestsIndex == -1 && networkIndex == -1 && variablesIndex == -1 { + return + } + if networkIndex != -1 { + requestsIndex = networkIndex + } + if variablesIndex != -1 { + requestsIndex = variablesIndex + } + infoBlockData := infoBlockIndexData[:requestsIndex] + infoBlockClean := strings.TrimRight(infoBlockData, "\n") + + infoBlock := InfoBlock{} + err = yaml.Unmarshal([]byte(data), &infoBlock) + if err != nil { + gologger.Warning().Msgf("Could not unmarshal info block: %s\n", err) + } + + var changed bool + if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" { + changed = true + infoBlock.Info.Severity = newSeverity + gologger.Info().Msgf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore) + } + isCvssEmpty := cvssScore == 0 || cvssMetrics == "" + hasCvssChanged := infoBlock.Info.Classification.CvssScore != cvssScore || cvssMetrics != infoBlock.Info.Classification.CvssMetrics + if !isCvssEmpty && hasCvssChanged { + changed = true + infoBlock.Info.Classification.CvssMetrics = cvssMetrics + infoBlock.Info.Classification.CvssScore = cvssScore + infoBlock.Info.Classification.CveId = cveName + if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") { + infoBlock.Info.Classification.CweId = strings.Join(cweID, ",") + } + } + // If there is no description field, fill the description from CVE information + enDescription, err := getEnglishLangString(cveItem.Cve.Descriptions) + hasDescriptionData := err != nil + isDescriptionEmpty := infoBlock.Info.Description == "" + if isDescriptionEmpty && hasDescriptionData { + changed = true + // removes all new lines + description := stringsutil.ReplaceAll(enDescription, "", "\n", "\\", "'", "\t") + description += "\n" + infoBlock.Info.Description = description + } + + // we are unmarshaling info block to have valid data + var referenceDataURLs []string + + // skip sites that are no longer alive + for _, reference := range cveItem.Cve.References { + if stringsutil.ContainsAny(reference.URL, badRefs...) { + continue + } + referenceDataURLs = append(referenceDataURLs, reference.URL) + } + hasReferenceData := len(cveItem.Cve.References) > 0 + areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs) + referencesCount := len(infoBlock.Info.Reference) + if hasReferenceData && !areCveReferencesContained { + changed = true + for _, ref := range referenceDataURLs { + referencesCount++ + if referencesCount >= maxReferenceCount { + break + } + infoBlock.Info.Reference = append(infoBlock.Info.Reference, ref) + } + infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference)) + } + + cpeSet := map[string]bool{} + for _, config := range cveItem.Cve.Configurations { + // Right now this covers only simple configurations. More complex configurations can have multiple CPEs + if len(config.Nodes) == 1 { + changed = true + node := config.Nodes[0] + for _, match := range node.CpeMatch { + cpeSet[extractVersionlessCpe(match.Criteria)] = true + } + } + } + uniqueCpes := make([]string, 0, len(cpeSet)) + for k := range cpeSet { + uniqueCpes = append(uniqueCpes, k) + } + if len(uniqueCpes) == 1 { + infoBlock.Info.Classification.Cpe = uniqueCpes[0] + } + + epss, err := fetchEpss(cveName) + if err != nil { + log.Printf("Could not fetch Epss score: %s\n", err) + return + } + hasEpssChanged := epss != infoBlock.Info.Classification.EpssScore + if hasEpssChanged { + changed = true + infoBlock.Info.Classification.EpssScore = epss + } + + var newInfoBlock bytes.Buffer + yamlEncoder := yaml.NewEncoder(&newInfoBlock) + yamlEncoder.SetIndent(yamlIndentSpaces) + err = yamlEncoder.Encode(infoBlock) + if err != nil { + gologger.Warning().Msgf("Could not marshal info block: %s\n", err) + return + } + newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") + + newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlockData) + if changed { + _ = os.WriteFile(filePath, []byte(newTemplate), 0644) + gologger.Info().Msgf("Wrote updated template to %s\n", filePath) + } +} + +func getPrimaryCVSSData(vuln nvd.Vulnerability) (nvd.CvssData, error) { + for _, data := range vuln.Cve.Metrics.CvssMetricV31 { + if data.Type == "Primary" { + return data.CvssData, nil + } + } + for _, data := range vuln.Cve.Metrics.CvssMetricV3 { + if data.Type == "Primary" { + return data.CvssData, nil + } + } + return nvd.CvssData{}, fmt.Errorf("no primary cvss metric found") +} + +func getEnglishLangString(data []nvd.LangString) (string, error) { + for _, item := range data { + if item.Lang == "en" { + return item.Value, nil + } + } + return "", fmt.Errorf("no english item found") +} + +func isSeverityMatchingCvssScore(severity string, score float64) string { + if score == 0.0 { + return "" + } + var expected string + + if score >= 0.1 && score <= 3.9 { + expected = "low" + } else if score >= 4.0 && score <= 6.9 { + expected = "medium" + } else if score >= 7.0 && score <= 8.9 { + expected = "high" + } else if score >= 9.0 && score <= 10.0 { + expected = "critical" + } + if expected != "" && expected != severity { + return expected + } + return "" +} + +func extractVersionlessCpe(cpe string) string { + parts := strings.Split(cpe, ":") + versionlessPart := parts[0:5] + rest := strings.Split(strings.Repeat("*", len(parts)-len(versionlessPart)), "") + return strings.Join(append(versionlessPart, rest...), ":") +} + +type ApiFirstEpssResponse struct { + Status string `json:"status"` + StatusCode int `json:"status-code"` + Version string `json:"version"` + Access string `json:"access"` + Total int `json:"total"` + Offset int `json:"offset"` + Limit int `json:"limit"` + Data []struct { + Cve string `json:"cve"` + Epss string `json:"epss"` + Percentile string `json:"percentile"` + Date string `json:"date"` + } `json:"data"` +} + +func fetchEpss(cveId string) (float64, error) { + resp, err := http.Get(fmt.Sprintf("https://api.first.org/data/v1/epss?cve=%s", cveId)) + if err != nil { + return 0, fmt.Errorf("unable to fetch EPSS data from first.org: %v", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, fmt.Errorf("unable to read reponse body: %v", err) + } + var parsedResp ApiFirstEpssResponse + err = json.Unmarshal(body, &parsedResp) + if err != nil { + return 0, fmt.Errorf("error while parsing EPSS response: %v", err) + } + if len(parsedResp.Data) != 1 { + return 0, fmt.Errorf("unexpected number of results in EPSS response. Expecting exactly 1, got %v", len(parsedResp.Data)) + } + epss := parsedResp.Data[0].Epss + return strconv.ParseFloat(epss, 64) +} + +type cisaKEVData struct { + Vulnerabilities []struct { + CVEID string `json:"cveID"` + } +} + +// fetchCISAKnownExploitedVulnerabilities fetches CISA known exploited +// vulnerabilities catalog for template tag enrichment +func fetchCISAKnownExploitedVulnerabilities() error { + data := &cisaKEVData{} + + resp, err := retryablehttp.DefaultClient().Get("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json") + if err != nil { + return errors.Wrap(err, "could not get cisa kev catalog") + } + defer resp.Body.Close() + + if err := json.NewDecoder(resp.Body).Decode(data); err != nil { + return errors.Wrap(err, "could not decode cisa kev catalog json data") + } + cisaKnownExploitedVulnerabilities = make(map[string]struct{}) + for _, vuln := range data.Vulnerabilities { + cisaKnownExploitedVulnerabilities[strings.ToLower(vuln.CVEID)] = struct{}{} + } + return nil +} + +// parseAndAddCISAKevTagTemplate parses and adds `kev` tag to CISA KEV templates. +// also removes cisa tag if it exists +func parseAndAddCISAKevTagTemplate(path string, data string) (string, error) { + block := &InfoBlock{} + + if err := yaml.NewDecoder(strings.NewReader(data)).Decode(block); err != nil { + return "", errors.Wrap(err, "could not decode template yaml") + } + splitted := strings.Split(block.Info.Tags, ",") + if len(splitted) == 0 { + return data, nil + } + + var cisaIndex = -1 + for i, tag := range splitted { + // If we already have tag, return + if tag == "kev" { + return data, nil + } + if tag == "cisa" { + cisaIndex = i + } + } + // Remove CISA index tag element + if cisaIndex >= 0 { + splitted = append(splitted[:cisaIndex], splitted[cisaIndex+1:]...) + } + splitted = append(splitted, "kev") + replaced := strings.ReplaceAll(data, block.Info.Tags, strings.Join(splitted, ",")) + return replaced, os.WriteFile(path, []byte(replaced), os.ModePerm) +} + +// parseAndAddReferenceBasedTags parses and adds reference based tags to templates +func parseAndAddReferenceBasedTags(path string, data string) (string, error) { + block := &InfoBlock{} + if err := yaml.NewDecoder(strings.NewReader(data)).Decode(block); err != nil { + return "", errors.Wrap(err, "could not decode template yaml") + } + splitted := strings.Split(block.Info.Tags, ",") + if len(splitted) == 0 { + return data, nil + } + tagsCurrent := fmt.Sprintf("tags: %s", block.Info.Tags) + newTags := suggestTagsBasedOnReference(block.Info.Reference, splitted) + + if len(newTags) == len(splitted) { + return data, nil + } + replaced := strings.ReplaceAll(data, tagsCurrent, fmt.Sprintf("tags: %s", strings.Join(newTags, ","))) + return replaced, os.WriteFile(path, []byte(replaced), os.ModePerm) +} + +var referenceMapping = map[string]string{ + "huntr.dev": "huntr", + "hackerone.com": "hackerone", + "tenable.com": "tenable", + "packetstormsecurity.org": "packetstorm", + "seclists.org": "seclists", + "wpscan.com": "wpscan", + "packetstormsecurity.com": "packetstorm", + "exploit-db.com": "edb", + "https://github.com/rapid7/metasploit-framework/": "msf", + "https://github.com/vulhub/vulhub/": "vulhub", +} + +func suggestTagsBasedOnReference(references, currentTags []string) []string { + uniqueTags := make(map[string]struct{}) + for _, value := range currentTags { + uniqueTags[value] = struct{}{} + } + + for _, reference := range references { + parsed, err := url.Parse(reference) + if err != nil { + continue + } + hostname := parsed.Hostname() + + for value, tag := range referenceMapping { + if strings.HasSuffix(hostname, value) || strings.HasPrefix(reference, value) { + uniqueTags[tag] = struct{}{} + } + } + } + newTags := make([]string, 0, len(uniqueTags)) + for tag := range uniqueTags { + newTags = append(newTags, tag) + } + return newTags +} + +// InfoBlock Cloning struct from nuclei as we don't want any validation +type InfoBlock struct { + Info TemplateInfo `yaml:"info"` +} + +type TemplateClassification struct { + CvssMetrics string `yaml:"cvss-metrics,omitempty"` + CvssScore float64 `yaml:"cvss-score,omitempty"` + CveId string `yaml:"cve-id,omitempty"` + CweId string `yaml:"cwe-id,omitempty"` + Cpe string `yaml:"cpe,omitempty"` + EpssScore float64 `yaml:"epss-score,omitempty"` +} + +type TemplateInfo struct { + Name string `yaml:"name"` + Author string `yaml:"author"` + Severity string `yaml:"severity,omitempty"` + Description string `yaml:"description,omitempty"` + Reference []string `yaml:"reference,omitempty"` + Remediation string `yaml:"remediation,omitempty"` + Classification TemplateClassification `yaml:"classification,omitempty"` + Metadata map[string]interface{} `yaml:"metadata,omitempty"` + Tags string `yaml:"tags,omitempty"` +} + +// parseAndAddMaxRequests parses and adds max requests to templates +func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, error) { + template, err := parseTemplate(catalog, path) + if err != nil { + gologger.Warning().Label("max-request").Msgf("Could not parse template: %s\n", err) + return data, err + } + + if template.TotalRequests < 1 { + return data, nil + } + // Marshal the updated info block back to YAML. + infoBlockStart, infoBlockEnd := getInfoStartEnd(data) + infoBlockOrig := data[infoBlockStart:infoBlockEnd] + infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n") + + infoBlock := InfoBlock{} + err = yaml.Unmarshal([]byte(data), &infoBlock) + if err != nil { + gologger.Warning().Label("max-request").Msgf("Could not unmarshal info block: %s\n", err) + return data, err + } + // if metadata is nil, create a new map + if infoBlock.Info.Metadata == nil { + infoBlock.Info.Metadata = make(map[string]interface{}) + } + // do not update if it is already present and equal + if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests { + return data, nil + } + infoBlock.Info.Metadata["max-request"] = template.TotalRequests + + var newInfoBlock bytes.Buffer + yamlEncoder := yaml.NewEncoder(&newInfoBlock) + yamlEncoder.SetIndent(yamlIndentSpaces) + err = yamlEncoder.Encode(infoBlock) + if err != nil { + gologger.Warning().Msgf("Could not marshal info block: %s\n", err) + return data, err + } + newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") + + // replace old info block with new info block + newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData) + + err = os.WriteFile(path, []byte(newTemplate), 0644) + if err == nil { + gologger.Info().Label("max-request").Msgf("Wrote updated template to %s\n", path) + } + return newTemplate, err +} + +// parseTemplate parses a template and returns the template object +func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) { + executorOpts := protocols.ExecutorOptions{ + Catalog: catalog, + Options: defaultOpts, + } + reader, err := executorOpts.Catalog.OpenFile(templatePath) + if err != nil { + return nil, err + } + template, err := templates.ParseTemplateFromReader(reader, nil, executorOpts) + if err != nil { + return nil, err + } + return template, nil +} + +// find the start and end of the info block +func getInfoStartEnd(data string) (int, int) { + info := strings.Index(data, "info:") + var indices []int + for _, re := range allTagsRegex { + // find the first occurrence of the label + match := re.FindStringIndex(data) + if match != nil { + indices = append(indices, match[0]) + } + } + // find the first one after info block + sort.Ints(indices) + return info, indices[0] - 1 +} diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 1fa04148fe..697486b0a4 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -89,7 +89,3 @@ {{binary}} -id tls-version -u scanme.sh # host:port {{binary}} -id tls-version -u scanme.sh:22 - -# Options -# Tls Impersonate -{{binary}} -id tech-detect -tlsi -u https://scanme.sh \ No newline at end of file diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 28d1d534f1..6031877487 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -210,7 +210,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"), flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"), flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"), - flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), ) flagSet.CreateGroup("interactsh", "interactsh", diff --git a/v2/cmd/nuclei/test.yaml b/v2/cmd/nuclei/test.yaml deleted file mode 100644 index ecb62d2ec6..0000000000 --- a/v2/cmd/nuclei/test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: basic-example - -info: - name: Test HTTP Template - author: pdteam - severity: info - -http: - - raw: - - |+ - GET / HTTP/1.1 - Host: {{Hostname}} - - unsafe: true - matchers: - - type: dsl - dsl: - - true \ No newline at end of file diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go deleted file mode 100644 index 97f8d6122b..0000000000 --- a/v2/cmd/tmc/main.go +++ /dev/null @@ -1,435 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "log" - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "strings" - - "github.com/projectdiscovery/goflags" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/retryablehttp-go" - errorutil "github.com/projectdiscovery/utils/errors" - "gopkg.in/yaml.v3" -) - -const ( - yamlIndentSpaces = 2 - // templateman api base url - tmBaseUrlDefault = "https://tm.nuclei.sh" -) - -var tmBaseUrl string - -func init() { - tmBaseUrl = os.Getenv("TEMPLATEMAN_SERVER") - if tmBaseUrl == "" { - tmBaseUrl = tmBaseUrlDefault - } -} - -// allTagsRegex is a list of all tags in nuclei templates except id, info, and - -var allTagsRegex []*regexp.Regexp -var defaultOpts = types.DefaultOptions() - -func init() { - var tm templates.Template - t := reflect.TypeOf(tm) - for i := 0; i < t.NumField(); i++ { - tag := t.Field(i).Tag.Get("yaml") - if strings.Contains(tag, ",") { - tag = strings.Split(tag, ",")[0] - } - // ignore these tags - if tag == "id" || tag == "info" || tag == "" || tag == "-" { - continue - } - re := regexp.MustCompile(tag + `:\s*\n`) - if t.Field(i).Type.Kind() == reflect.Bool { - re = regexp.MustCompile(tag + `:\s*(true|false)\s*\n`) - } - allTagsRegex = append(allTagsRegex, re) - } - - defaultOpts := types.DefaultOptions() - // need to set headless to true for headless templates - defaultOpts.Headless = true - if err := protocolstate.Init(defaultOpts); err != nil { - gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) - } - if err := protocolinit.Init(defaultOpts); err != nil { - gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) - } -} - -var idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)") - -type options struct { - input string - errorLogFile string - lint bool - validate bool - format bool - enhance bool - maxRequest bool - debug bool -} - -func main() { - opts := options{} - flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`TemplateMan CLI is baisc utility built on the TemplateMan API to standardize nuclei templates.`) - - flagSet.CreateGroup("Input", "input", - flagSet.StringVarP(&opts.input, "input", "i", "", "Templates to annotate"), - ) - - flagSet.CreateGroup("Config", "config", - flagSet.BoolVarP(&opts.lint, "lint", "l", false, "lint given nuclei template"), - flagSet.BoolVarP(&opts.validate, "validate", "v", false, "validate given nuclei template"), - flagSet.BoolVarP(&opts.format, "format", "f", false, "format given nuclei template"), - flagSet.BoolVarP(&opts.enhance, "enhance", "e", false, "enhance given nuclei template"), - flagSet.BoolVarP(&opts.maxRequest, "max-request", "mr", false, "add / update max request counter"), - flagSet.StringVarP(&opts.errorLogFile, "error-log", "el", "", "file to write failed template update"), - flagSet.BoolVarP(&opts.debug, "debug", "d", false, "show debug message"), - ) - - if err := flagSet.Parse(); err != nil { - gologger.Fatal().Msgf("Error parsing flags: %s\n", err) - } - - if opts.input == "" { - gologger.Fatal().Msg("input template path/directory is required") - } - if strings.HasPrefix(opts.input, "~/") { - home, err := os.UserHomeDir() - if err != nil { - log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err) - } - opts.input = filepath.Join(home, (opts.input)[2:]) - } - gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) - if opts.debug { - gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) - } - if err := process(opts); err != nil { - gologger.Error().Msgf("could not process: %s\n", err) - } -} - -func process(opts options) error { - tempDir, err := os.MkdirTemp("", "nuclei-nvd-%s") - if err != nil { - return err - } - defer os.RemoveAll(tempDir) - - var errFile *os.File - if opts.errorLogFile != "" { - errFile, err = os.OpenFile(opts.errorLogFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - gologger.Fatal().Msgf("could not open error log file: %s\n", err) - } - defer errFile.Close() - } - - templateCatalog := disk.NewCatalog(filepath.Dir(opts.input)) - paths, err := templateCatalog.GetTemplatePath(opts.input) - if err != nil { - return err - } - for _, path := range paths { - data, err := os.ReadFile(path) - if err != nil { - return err - } - dataString := string(data) - - if opts.maxRequest { - var updated bool // if max-requests is updated - dataString, updated, err = parseAndAddMaxRequests(templateCatalog, path, dataString) - if err != nil { - gologger.Info().Label("max-request").Msgf(logErrMsg(path, err, opts.debug, errFile)) - } else { - if updated { - gologger.Info().Label("max-request").Msgf("✅ updated template: %s\n", path) - } - // do not print if max-requests is not updated - } - } - - if opts.lint { - lint, err := lintTemplate(dataString) - if err != nil { - gologger.Info().Label("lint").Msg(logErrMsg(path, err, opts.debug, errFile)) - } - if lint { - gologger.Info().Label("lint").Msgf("✅ lint template: %s\n", path) - } - } - - if opts.validate { - validate, err := validateTemplate(dataString) - if err != nil { - gologger.Info().Label("validate").Msg(logErrMsg(path, err, opts.debug, errFile)) - } - if validate { - gologger.Info().Label("validate").Msgf("✅ validated template: %s\n", path) - } - } - - if opts.format { - formatedTemplateData, isFormated, err := formatTemplate(dataString) - if err != nil { - gologger.Info().Label("format").Msg(logErrMsg(path, err, opts.debug, errFile)) - } else { - if isFormated { - _ = os.WriteFile(path, []byte(formatedTemplateData), 0644) - dataString = formatedTemplateData - gologger.Info().Label("format").Msgf("✅ formated template: %s\n", path) - } - } - } - - if opts.enhance { - // currently enhance api only supports cve-id's - matches := idRegex.FindAllStringSubmatch(dataString, 1) - if len(matches) == 0 { - continue - } - enhancedTemplateData, isEnhanced, err := enhanceTemplate(dataString) - if err != nil { - gologger.Info().Label("enhance").Msg(logErrMsg(path, err, opts.debug, errFile)) - continue - } else { - if isEnhanced { - _ = os.WriteFile(path, []byte(enhancedTemplateData), 0644) - gologger.Info().Label("enhance").Msgf("✅ updated template: %s\n", path) - } - } - } - } - return nil -} - -func logErrMsg(path string, err error, debug bool, errFile *os.File) string { - msg := fmt.Sprintf("❌ template: %s\n", path) - if debug { - msg = fmt.Sprintf("❌ template: %s err: %s\n", path, err) - } - if errFile != nil { - _, _ = errFile.WriteString(fmt.Sprintf("❌ template: %s err: %s\n", path, err)) - } - return msg -} - -// enhanceTemplateData enhances template data using templateman -// ref: https://github.com/projectdiscovery/templateman/blob/main/templateman-rest-api/README.md#enhance-api -func enhanceTemplate(data string) (string, bool, error) { - resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/enhance", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) - if err != nil { - return data, false, err - } - if resp.StatusCode != 200 { - return data, false, errorutil.New("unexpected status code: %v", resp.Status) - } - var templateResp TemplateResp - if err := json.NewDecoder(resp.Body).Decode(&templateResp); err != nil { - return data, false, err - } - if strings.TrimSpace(templateResp.Enhanced) != "" { - return templateResp.Enhanced, templateResp.Enhance, nil - } - if templateResp.ValidateErrorCount > 0 { - if len(templateResp.ValidateError) > 0 { - return data, false, errorutil.NewWithTag("validate", templateResp.ValidateError[0].Message+": at line %v", templateResp.ValidateError[0].Mark.Line) - } - return data, false, errorutil.New("validation failed").WithTag("validate") - } - if templateResp.Error.Name != "" { - return data, false, errorutil.New(templateResp.Error.Name) - } - if strings.TrimSpace(templateResp.Enhanced) == "" && !templateResp.Lint { - if templateResp.LintError.Reason != "" { - return data, false, errorutil.NewWithTag("lint", templateResp.LintError.Reason+" : at line %v", templateResp.LintError.Mark.Line) - } - return data, false, errorutil.NewWithTag("lint", "at line: %v", templateResp.LintError.Mark.Line) - } - return data, false, errorutil.New("template enhance failed") -} - -// formatTemplateData formats template data using templateman format api -func formatTemplate(data string) (string, bool, error) { - resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/format", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) - if err != nil { - return data, false, err - } - if resp.StatusCode != 200 { - return data, false, errorutil.New("unexpected status code: %v", resp.Status) - } - var templateResp TemplateResp - if err := json.NewDecoder(resp.Body).Decode(&templateResp); err != nil { - return data, false, err - } - if strings.TrimSpace(templateResp.Updated) != "" { - return templateResp.Updated, templateResp.Format, nil - } - if templateResp.ValidateErrorCount > 0 { - if len(templateResp.ValidateError) > 0 { - return data, false, errorutil.NewWithTag("validate", templateResp.ValidateError[0].Message+": at line %v", templateResp.ValidateError[0].Mark.Line) - } - return data, false, errorutil.New("validation failed").WithTag("validate") - } - if templateResp.Error.Name != "" { - return data, false, errorutil.New(templateResp.Error.Name) - } - if strings.TrimSpace(templateResp.Updated) == "" && !templateResp.Lint { - if templateResp.LintError.Reason != "" { - return data, false, errorutil.NewWithTag("lint", templateResp.LintError.Reason+" : at line %v", templateResp.LintError.Mark.Line) - } - return data, false, errorutil.NewWithTag("lint", "at line: %v", templateResp.LintError.Mark.Line) - } - return data, false, errorutil.New("template format failed") -} - -// lintTemplateData lints template data using templateman lint api -func lintTemplate(data string) (bool, error) { - resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/lint", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) - if err != nil { - return false, err - } - if resp.StatusCode != 200 { - return false, errorutil.New("unexpected status code: %v", resp.Status) - } - var lintResp TemplateLintResp - if err := json.NewDecoder(resp.Body).Decode(&lintResp); err != nil { - return false, err - } - if lintResp.Lint { - return true, nil - } - if lintResp.LintError.Reason != "" { - return false, errorutil.NewWithTag("lint", lintResp.LintError.Reason+" : at line %v", lintResp.LintError.Mark.Line) - } - return false, errorutil.NewWithTag("lint", "at line: %v", lintResp.LintError.Mark.Line) -} - -// validateTemplate validates template data using templateman validate api -func validateTemplate(data string) (bool, error) { - resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/validate", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) - if err != nil { - return false, err - } - if resp.StatusCode != 200 { - return false, errorutil.New("unexpected status code: %v", resp.Status) - } - var validateResp TemplateResp - if err := json.NewDecoder(resp.Body).Decode(&validateResp); err != nil { - return false, err - } - if validateResp.Validate { - return true, nil - } - if validateResp.ValidateErrorCount > 0 { - if len(validateResp.ValidateError) > 0 { - return false, errorutil.NewWithTag("validate", validateResp.ValidateError[0].Message+": at line %v", validateResp.ValidateError[0].Mark.Line) - } - return false, errorutil.New("validation failed").WithTag("validate") - } - if validateResp.Error.Name != "" { - return false, errorutil.New(validateResp.Error.Name) - } - return false, errorutil.New("template validation failed") -} - -// parseAndAddMaxRequests parses and adds max requests to templates -func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, bool, error) { - template, err := parseTemplate(catalog, path) - if err != nil { - return data, false, err - } - if template.TotalRequests < 1 { - return data, false, nil - } - // Marshal the updated info block back to YAML. - infoBlockStart, infoBlockEnd := getInfoStartEnd(data) - infoBlockOrig := data[infoBlockStart:infoBlockEnd] - infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n") - infoBlock := InfoBlock{} - err = yaml.Unmarshal([]byte(data), &infoBlock) - if err != nil { - return data, false, err - } - // if metadata is nil, create a new map - if infoBlock.Info.Metadata == nil { - infoBlock.Info.Metadata = make(map[string]interface{}) - } - // do not update if it is already present and equal - if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests { - return data, false, nil - } - infoBlock.Info.Metadata["max-request"] = template.TotalRequests - - var newInfoBlock bytes.Buffer - yamlEncoder := yaml.NewEncoder(&newInfoBlock) - yamlEncoder.SetIndent(yamlIndentSpaces) - err = yamlEncoder.Encode(infoBlock) - if err != nil { - return data, false, err - } - newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") - // replace old info block with new info block - newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData) - err = os.WriteFile(path, []byte(newTemplate), 0644) - if err == nil { - return newTemplate, true, nil - } - return newTemplate, false, err -} - -// parseTemplate parses a template and returns the template object -func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) { - executorOpts := protocols.ExecutorOptions{ - Catalog: catalog, - Options: defaultOpts, - } - reader, err := executorOpts.Catalog.OpenFile(templatePath) - if err != nil { - return nil, err - } - template, err := templates.ParseTemplateFromReader(reader, nil, executorOpts) - if err != nil { - return nil, err - } - return template, nil -} - -// find the start and end of the info block -func getInfoStartEnd(data string) (int, int) { - info := strings.Index(data, "info:") - var indices []int - for _, re := range allTagsRegex { - // find the first occurrence of the label - match := re.FindStringIndex(data) - if match != nil { - indices = append(indices, match[0]) - } - } - // find the first one after info block - sort.Ints(indices) - return info, indices[0] - 1 -} diff --git a/v2/cmd/tmc/types.go b/v2/cmd/tmc/types.go deleted file mode 100644 index 0587464097..0000000000 --- a/v2/cmd/tmc/types.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -type Mark struct { - Name string `json:"name,omitempty"` - Position int `json:"position,omitempty"` - Line int `json:"line,omitempty"` - Column int `json:"column,omitempty"` - Snippet string `json:"snippet,omitempty"` -} - -type Error struct { - Name string `json:"name"` - Mark Mark `json:"mark"` -} - -type LintError struct { - Name string `json:"name,omitempty"` - Reason string `json:"reason,omitempty"` - Mark Mark `json:"mark,omitempty"` -} - -type TemplateLintResp struct { - Input string `json:"template_input,omitempty"` - Lint bool `json:"template_lint,omitempty"` - LintError LintError `json:"lint_error,omitempty"` -} - -type ValidateError struct { - Location string `json:"location,omitempty"` - Message string `json:"message,omitempty"` - Name string `json:"name,omitempty"` - Argument interface{} `json:"argument,omitempty"` - Stack string `json:"stack,omitempty"` - Mark struct { - Line int `json:"line,omitempty"` - Column int `json:"column,omitempty"` - Pos int `json:"pos,omitempty"` - } `json:"mark,omitempty"` -} - -// TemplateResponse from templateman to be used for enhancing and formatting -type TemplateResp struct { - Input string `json:"template_input,omitempty"` - Format bool `json:"template_format,omitempty"` - Updated string `json:"updated_template,omitempty"` - Enhance bool `json:"template_enhance,omitempty"` - Enhanced string `json:"enhanced_template,omitempty"` - Lint bool `json:"template_lint,omitempty"` - LintError LintError `json:"lint_error,omitempty"` - Validate bool `json:"template_validate,omitempty"` - ValidateErrorCount int `json:"validate_error_count,omitempty"` - ValidateError []ValidateError `json:"validate_error,omitempty"` - Error Error `json:"error,omitempty"` -} - -// InfoBlock Cloning struct from nuclei as we don't want any validation -type InfoBlock struct { - Info TemplateInfo `yaml:"info"` -} - -type TemplateClassification struct { - CvssMetrics string `yaml:"cvss-metrics,omitempty"` - CvssScore float64 `yaml:"cvss-score,omitempty"` - CveId string `yaml:"cve-id,omitempty"` - CweId string `yaml:"cwe-id,omitempty"` - Cpe string `yaml:"cpe,omitempty"` - EpssScore float64 `yaml:"epss-score,omitempty"` -} - -type TemplateInfo struct { - Name string `yaml:"name"` - Author string `yaml:"author"` - Severity string `yaml:"severity,omitempty"` - Description string `yaml:"description,omitempty"` - Reference interface{} `yaml:"reference,omitempty"` - Remediation string `yaml:"remediation,omitempty"` - Classification TemplateClassification `yaml:"classification,omitempty"` - Metadata map[string]interface{} `yaml:"metadata,omitempty"` - Tags string `yaml:"tags,omitempty"` -} diff --git a/v2/go.mod b/v2/go.mod index 9719a4cf83..4b13cce840 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -17,15 +17,15 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.55 + github.com/miekg/dns v1.1.54 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/clistats v0.0.18 - github.com/projectdiscovery/fastdialer v0.0.31 + github.com/projectdiscovery/clistats v0.0.12 + github.com/projectdiscovery/fastdialer v0.0.29 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.13 - github.com/projectdiscovery/retryabledns v1.0.30 + github.com/projectdiscovery/retryabledns v1.0.29 github.com/projectdiscovery/retryablehttp-go v1.0.17 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 @@ -66,18 +66,19 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd + github.com/projectdiscovery/dsl v0.0.9 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/httpx v1.3.0 github.com/projectdiscovery/mapcidr v1.1.2 + github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 github.com/projectdiscovery/ratelimit v0.0.8 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.0 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.39-0.20230621170112-8dd2c290d962 + github.com/projectdiscovery/utils v0.0.38 github.com/projectdiscovery/wappalyzergo v0.0.94 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -126,9 +127,9 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/projectdiscovery/asnmap v1.0.4 // indirect github.com/projectdiscovery/cdncheck v1.0.6 // indirect - github.com/projectdiscovery/freeport v0.0.5 // indirect + github.com/projectdiscovery/freeport v0.0.4 // indirect github.com/refraction-networking/utls v1.3.2 // indirect - github.com/sashabaranov/go-openai v1.11.2 // indirect + github.com/sashabaranov/go-openai v1.9.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 8a491f1146..41edc78b0e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -31,6 +31,7 @@ github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:AqtPw7WNT0O69k+AbPKWVGYeW94TqgMW/g+Ppc8AZr4= github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek= github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= @@ -53,6 +54,7 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= @@ -67,6 +69,7 @@ github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= @@ -352,8 +355,9 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= -github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= @@ -403,16 +407,16 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6uGfPofVpC4c= github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= -github.com/projectdiscovery/clistats v0.0.18 h1:WLQNqLXsKvjoieDwXJO/1jlnxR0x9vdFaRUAR3gXfKQ= -github.com/projectdiscovery/clistats v0.0.18/go.mod h1:YUnUrMHFw+FHwUTIKr1KDUwz81x+SFjPU3xfLqXfzf0= -github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd h1:16DMjd4HeACrC9CkWJkkLeSh+LYPDorwNx11BlTbonU= -github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd/go.mod h1:S72Cq/lfxzkldf64Sul1G2KFbGKNgpRFFCF/FazpznM= -github.com/projectdiscovery/fastdialer v0.0.31 h1:eu0wTBCWjT8dXChmBtnQaAxoFpkLdvq0VroRxZoe/M8= -github.com/projectdiscovery/fastdialer v0.0.31/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= +github.com/projectdiscovery/clistats v0.0.12 h1:KLYJxpiwEFidduU4PbcwEcCQ2L7c5wrf7DI5IN5fZ+8= +github.com/projectdiscovery/clistats v0.0.12/go.mod h1:9luKJj+7Hjq3+a7g129sKWRYx4SbTdkUWZQxabn3H5Y= +github.com/projectdiscovery/dsl v0.0.9 h1:VfznBxpbNKMn2amQd9gtRnMfK1/Sf9MwsJD9x2Et/fY= +github.com/projectdiscovery/dsl v0.0.9/go.mod h1:kdPdbbqceWxkSedXm99z0Hzh9z/DFj42A9L95GJjybo= +github.com/projectdiscovery/fastdialer v0.0.29 h1:uDy2/bXHl8ISkuRp0EpmajkfWHewL3q5oDcYxB07ME8= +github.com/projectdiscovery/fastdialer v0.0.29/go.mod h1:CBzmr7QS+Ml66h1jjuudR8Uzl6bt2YeqYmTg0IedWsI= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= -github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= -github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= +github.com/projectdiscovery/freeport v0.0.4 h1:H4VrK/7hUcC1zbg46zv9iSMBACBDpUqcHkV+FUyXISw= +github.com/projectdiscovery/freeport v0.0.4/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goflags v0.1.10 h1:Gompf8JDy8y+5c4eWlc70KKtPuDH/hqFB3tMeHcMiKk= github.com/projectdiscovery/goflags v0.1.10/go.mod h1:MHEkqm3XgxBf5fK4gr3IXsj6VeLTq4qJYGC/4JRYQ74= github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoii4P2yDovqck= @@ -427,14 +431,16 @@ github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4 github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= +github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 h1:aDq18tNWbnN5ZM0ADQb+8KB4DEPIGZMXdDmcXyFUoNg= +github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8/go.mod h1:JiVXOIewstCBMPsO+ZnmI43UXMPJGEE1jwuFVz4ujKM= github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= github.com/projectdiscovery/rawhttp v0.1.13 h1:Xn3NY3SYIk0151K5Qfuvx3tayl2UOoxMuVyYvGT95BA= github.com/projectdiscovery/rawhttp v0.1.13/go.mod h1:AjZUYdPCx4xqeWYPqFPLGCxQsVFeUrobxidnU6Nta8M= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0= -github.com/projectdiscovery/retryabledns v1.0.30/go.mod h1:+Aqc0TjKGcTtP0HtXE8o1GzrjAHhSno6hSF+L63TBtI= +github.com/projectdiscovery/retryabledns v1.0.29 h1:44EphLP5gRgVxlge9/qm5Gue+9cDd/BAILTF9PQQx54= +github.com/projectdiscovery/retryabledns v1.0.29/go.mod h1:NtbDTfcsW9hIUf0HuVQNZSTTG063Phy0uaBBjZlif0Q= github.com/projectdiscovery/retryablehttp-go v1.0.17 h1:oppnrypatWsHxcMU5RuAcUsUu3nxBhId2CF3OBj9XJA= github.com/projectdiscovery/retryablehttp-go v1.0.17/go.mod h1:zJh8bQdxhIsaEGnxsacvMbgiCKT4UAOr4T1kZBnSa68= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= @@ -444,8 +450,9 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.39-0.20230621170112-8dd2c290d962 h1:qQnIsYB72MmuaM9orhKpDzY0ddJKHf9Nuih0FnyV6x8= -github.com/projectdiscovery/utils v0.0.39-0.20230621170112-8dd2c290d962/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= +github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= +github.com/projectdiscovery/utils v0.0.38 h1:EIAgaP3imfcQY+laxNOU9LXh7VZNAbmiwXsQN0mAxdQ= +github.com/projectdiscovery/utils v0.0.38/go.mod h1:5+WAxSV7yGl6SDCtR1qiOyiEMCIo3jIff+A5OiYTCgM= github.com/projectdiscovery/wappalyzergo v0.0.94 h1:IVRskuU95MajWCKYgvH5L67+MXDOWJDWSeBD61OsS/A= github.com/projectdiscovery/wappalyzergo v0.0.94/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -461,10 +468,11 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.11.2 h1:HuMf+18eldSKbqVblyeCQbtcqSpGVfqTshvi8Bn6zes= -github.com/sashabaranov/go-openai v1.11.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.9.1 h1:3N52HkJKo9Zlo/oe1AVv5ZkCOny0ra58/ACvAxkN3MM= +github.com/sashabaranov/go-openai v1.9.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -591,6 +599,7 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= @@ -601,6 +610,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= @@ -626,6 +636,7 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -635,6 +646,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -648,11 +660,13 @@ golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -703,6 +717,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -715,6 +730,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= @@ -724,6 +740,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -738,6 +755,7 @@ golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= diff --git a/v2/pkg/core/executors.go b/v2/pkg/core/executors.go index 45013efaba..b1c9e67b94 100644 --- a/v2/pkg/core/executors.go +++ b/v2/pkg/core/executors.go @@ -78,13 +78,13 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target // skips indexes lower than the minimum in-flight at interruption time var skip bool if resumeFromInfo.Completed { // the template was completed - gologger.Debug().Msgf("[%s] Skipping \"%s\": Resume - Template already completed\n", template.ID, scannedValue.Input) + gologger.Debug().Msgf("[%s] Skipping \"%s\": Resume - Template already completed\n", template.ID, scannedValue) skip = true } else if index < resumeFromInfo.SkipUnder { // index lower than the sliding window (bulk-size) - gologger.Debug().Msgf("[%s] Skipping \"%s\": Resume - Target already processed\n", template.ID, scannedValue.Input) + gologger.Debug().Msgf("[%s] Skipping \"%s\": Resume - Target already processed\n", template.ID, scannedValue) skip = true } else if _, isInFlight := resumeFromInfo.InFlight[index]; isInFlight { // the target wasn't completed successfully - gologger.Debug().Msgf("[%s] Repeating \"%s\": Resume - Target wasn't completed\n", template.ID, scannedValue.Input) + gologger.Debug().Msgf("[%s] Repeating \"%s\": Resume - Target wasn't completed\n", template.ID, scannedValue) // skip is already false, but leaving it here for clarity skip = false } else if index > resumeFromInfo.DoAbove { // index above the sliding window (bulk-size) diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 014be676af..6ef593329c 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -99,23 +99,15 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64) p.stats.AddCounter("total", uint64(requestCount)) if p.active { - var printCallbackFunc clistats.DynamicCallback + var printCallbackFunc clistats.PrintCallback if p.outputJSON { printCallbackFunc = printCallbackJSON } else { printCallbackFunc = p.makePrintCallback() } - p.stats.AddDynamic("summary", printCallbackFunc) - if err := p.stats.Start(); err != nil { + if err := p.stats.Start(printCallbackFunc, p.tickDuration); err != nil { gologger.Warning().Msgf("Couldn't start statistics: %s", err) } - - p.stats.GetStatResponse(p.tickDuration, func(s string, err error) error { - if err != nil { - gologger.Warning().Msgf("Could not read statistics: %s\n", err) - } - return nil - }) } } @@ -153,8 +145,8 @@ func (p *StatsTicker) IncrementFailedRequestsBy(count int64) { p.stats.IncrementCounter("errors", int(count)) } -func (p *StatsTicker) makePrintCallback() func(stats clistats.StatisticsClient) interface{} { - return func(stats clistats.StatisticsClient) interface{} { +func (p *StatsTicker) makePrintCallback() func(stats clistats.StatisticsClient) { + return func(stats clistats.StatisticsClient) { builder := &strings.Builder{} var duration time.Duration @@ -217,16 +209,14 @@ func (p *StatsTicker) makePrintCallback() func(stats clistats.StatisticsClient) } fmt.Fprintf(os.Stderr, "%s", builder.String()) - return builder.String() } } -func printCallbackJSON(stats clistats.StatisticsClient) interface{} { +func printCallbackJSON(stats clistats.StatisticsClient) { builder := &strings.Builder{} if err := json.NewEncoder(builder).Encode(metricsMap(stats)); err == nil { fmt.Fprintf(os.Stderr, "%s", builder.String()) } - return builder.String() } func metricsMap(stats clistats.StatisticsClient) map[string]interface{} { diff --git a/v2/pkg/protocols/common/variables/variables.go b/v2/pkg/protocols/common/variables/variables.go index b806402aaa..980dedc25f 100644 --- a/v2/pkg/protocols/common/variables/variables.go +++ b/v2/pkg/protocols/common/variables/variables.go @@ -109,12 +109,11 @@ func evaluateVariableValue(expression string, values, processing map[string]inte // checkForLazyEval checks if the variables have any lazy evaluation i.e any dsl function // and sets the flag accordingly. func (variables *Variable) checkForLazyEval() bool { + variables.ForEach(func(key string, value interface{}) { - for _, v := range protocolutils.KnownVariables { - if stringsutil.ContainsAny(types.ToString(value), v) { - variables.LazyEval = true - return - } + if stringsutil.ContainsAny(types.ToString(value), protocolutils.KnownVariables...) { + variables.LazyEval = true + return } }) return variables.LazyEval diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 1195a05467..50f97c7dc7 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -11,7 +11,6 @@ import ( "golang.org/x/net/proxy" - "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -40,14 +39,9 @@ func newHttpClient(options *types.Options) (*http.Client, error) { } transport := &http.Transport{ - ForceAttemptHTTP2: options.ForceAttemptHTTP2, - DialContext: dialer.Dial, - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - if options.TlsImpersonate { - return dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil) - } - return dialer.DialTLS(ctx, network, addr) - }, + ForceAttemptHTTP2: options.ForceAttemptHTTP2, + DialContext: dialer.Dial, + DialTLSContext: dialer.DialTLS, MaxIdleConns: 500, MaxIdleConnsPerHost: 500, MaxConnsPerHost: 500, diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 6717b445c9..e0cb1a829c 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -17,7 +17,6 @@ import ( "golang.org/x/net/publicsuffix" "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -231,8 +230,6 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { if options.HasClientCertificates() { return Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig) - if options.TlsImpersonate { - return Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil) } return Dialer.DialTLS(ctx, network, addr) }, @@ -252,7 +249,6 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if proxyErr != nil { return nil, proxyErr } - dialer, err := proxy.FromURL(socksURL, proxy.Direct) if err != nil { return nil, err @@ -261,15 +257,16 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl dc := dialer.(interface { DialContext(ctx context.Context, network, addr string) (net.Conn, error) }) - - transport.DialContext = dc.DialContext - transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - // upgrade proxy connection to tls - conn, err := dc.DialContext(ctx, network, addr) - if err != nil { - return nil, err + if proxyErr == nil { + transport.DialContext = dc.DialContext + transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + // upgrade proxy connection to tls + conn, err := dc.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + return tls.Client(conn, tlsConfig), nil } - return tls.Client(conn, tlsConfig), nil } } diff --git a/v2/pkg/protocols/utils/variables.go b/v2/pkg/protocols/utils/variables.go index a94c99c995..41fd9529e9 100644 --- a/v2/pkg/protocols/utils/variables.go +++ b/v2/pkg/protocols/utils/variables.go @@ -8,51 +8,12 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - maputil "github.com/projectdiscovery/utils/maps" urlutil "github.com/projectdiscovery/utils/url" "github.com/weppos/publicsuffix-go/publicsuffix" ) // KnownVariables are the variables that are known to input requests -var KnownVariables maputil.Map[KnownVariable, string] - -func init() { - KnownVariables = maputil.Map[KnownVariable, string]{ - BaseURL: "BaseURL", - RootURL: "RootURL", - Hostname: "Hostname", - Host: "Host", - Port: "Port", - Path: "Path", - File: "File", - Scheme: "Scheme", - Input: "Input", - Fqdn: "FQDN", - Rdn: "RDN", - Dn: "DN", - Tld: "TLD", - Sd: "SD", - } -} - -type KnownVariable uint16 - -const ( - BaseURL KnownVariable = iota - RootURL - Hostname - Host - Port - Path - File - Scheme - Input - Fqdn - Rdn - Dn - Tld - Sd -) +var KnownVariables = []string{"BaseURL", "RootURL", "Hostname", "Host", "Port", "Path", "File", "Scheme", "Input", "FQDN", "RDN", "DN", "TLD", "SD"} // GenerateVariables will create default variables with context args func GenerateVariablesWithContextArgs(input *contextargs.Context, trailingSlash bool) map[string]interface{} { @@ -73,18 +34,18 @@ func GenerateDNSVariables(domain string) map[string]interface{} { domainName := strings.Join([]string{parsed.SLD, parsed.TLD}, ".") dnsVariables := make(map[string]interface{}) - for k, v := range KnownVariables { + for _, k := range KnownVariables { switch k { - case Fqdn: - dnsVariables[v] = domain - case Rdn: - dnsVariables[v] = domainName - case Dn: - dnsVariables[v] = parsed.SLD - case Tld: - dnsVariables[v] = parsed.TLD - case Sd: - dnsVariables[v] = parsed.TRD + case "FQDN": + dnsVariables[k] = domain + case "RDN": + dnsVariables[k] = domainName + case "DN": + dnsVariables[k] = parsed.SLD + case "TLD": + dnsVariables[k] = parsed.TLD + case "SD": + dnsVariables[k] = parsed.TRD } } return dnsVariables @@ -94,12 +55,13 @@ func GenerateDNSVariables(domain string) map[string]interface{} { // Returns the map of KnownVariables keys // This function is used by http, headless, websocket, network and whois protocols to generate protocol variables func GenerateVariables(input interface{}, removeTrailingSlash bool, additionalVars map[string]interface{}) map[string]interface{} { + var vars = make(map[string]interface{}) switch input := input.(type) { case string: parsed, err := urlutil.Parse(input) if err != nil { - return map[string]interface{}{KnownVariables[Input]: input, KnownVariables[Hostname]: input} + return map[string]interface{}{"Input": input, "Hostname": input} } vars = generateVariables(parsed, removeTrailingSlash) case *urlutil.URL: @@ -144,26 +106,26 @@ func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[stri } } knownVariables := make(map[string]interface{}) - for k, v := range KnownVariables { + for _, k := range KnownVariables { switch k { - case BaseURL: - knownVariables[v] = parsed.String() - case RootURL: - knownVariables[v] = fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) - case Hostname: - knownVariables[v] = parsed.Host - case Host: - knownVariables[v] = parsed.Hostname() - case Port: - knownVariables[v] = port - case Path: - knownVariables[v] = requestPath - case File: - knownVariables[v] = base - case Scheme: - knownVariables[v] = parsed.Scheme - case Input: - knownVariables[v] = parsed.String() + case "BaseURL": + knownVariables[k] = parsed.String() + case "RootURL": + knownVariables[k] = fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) + case "Hostname": + knownVariables[k] = parsed.Host + case "Host": + knownVariables[k] = parsed.Hostname() + case "Port": + knownVariables[k] = port + case "Path": + knownVariables[k] = requestPath + case "File": + knownVariables[k] = base + case "Scheme": + knownVariables[k] = parsed.Scheme + case "Input": + knownVariables[k] = parsed.String() } } return generators.MergeMaps(knownVariables, GenerateDNSVariables(parsed.Hostname())) diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 9924dcabb9..d323b9a8b5 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -7,13 +7,11 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" stringsutil "github.com/projectdiscovery/utils/strings" ) const indexFileName = "index.md" -const extension = ".md" type Exporter struct { directory string @@ -39,7 +37,9 @@ func New(options *Options) (*Exporter, error) { _ = os.MkdirAll(directory, 0755) // index generation header - dataHeader := util.CreateTableHeader("Hostname/IP", "Finding", "Severity") + dataHeader := "" + + "|Hostname/IP|Finding|Severity|\n" + + "|-|-|-|\n" err := os.WriteFile(filepath.Join(directory, indexFileName), []byte(dataHeader), 0644) if err != nil { @@ -51,34 +51,9 @@ func New(options *Options) (*Exporter, error) { // Export exports a passed result event to markdown func (exporter *Exporter) Export(event *output.ResultEvent) error { - // index file generation - file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer file.Close() - - filename := createFileName(event) - host := util.CreateLink(event.Host, filename) - finding := event.TemplateID + " " + event.MatcherName - severity := event.Info.SeverityHolder.Severity.String() - - _, err = file.WriteString(util.CreateTableRow(host, finding, severity)) - if err != nil { - return err - } - - dataBuilder := &bytes.Buffer{} - dataBuilder.WriteString(util.CreateHeading3(format.Summary(event))) - dataBuilder.WriteString("\n") - dataBuilder.WriteString(util.CreateHorizontalLine()) - dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{})) - data := dataBuilder.Bytes() - - return os.WriteFile(filepath.Join(exporter.directory, filename), data, 0644) -} + summary := format.Summary(event) + description := format.MarkdownDescription(event) -func createFileName(event *output.ResultEvent) string { filenameBuilder := &strings.Builder{} filenameBuilder.WriteString(event.TemplateID) filenameBuilder.WriteString("-") @@ -94,8 +69,29 @@ func createFileName(event *output.ResultEvent) string { filenameBuilder.WriteRune('-') filenameBuilder.WriteString(event.MatcherName) } - filenameBuilder.WriteString(extension) - return sanitizeFilename(filenameBuilder.String()) + filenameBuilder.WriteString(".md") + finalFilename := sanitizeFilename(filenameBuilder.String()) + + dataBuilder := &bytes.Buffer{} + dataBuilder.WriteString("### ") + dataBuilder.WriteString(summary) + dataBuilder.WriteString("\n---\n") + dataBuilder.WriteString(description) + data := dataBuilder.Bytes() + + // index generation + file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + _, err = file.WriteString("|[" + event.Host + "](" + finalFilename + ")" + "|" + event.TemplateID + " " + event.MatcherName + "|" + event.Info.SeverityHolder.Severity.String() + "|\n") + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(exporter.directory, finalFilename), data, 0644) } // Close closes the exporter after operation diff --git a/v2/pkg/reporting/exporters/markdown/util/markdown_formatter.go b/v2/pkg/reporting/exporters/markdown/util/markdown_formatter.go deleted file mode 100644 index 92b3b75b5d..0000000000 --- a/v2/pkg/reporting/exporters/markdown/util/markdown_formatter.go +++ /dev/null @@ -1,27 +0,0 @@ -package util - -import ( - "fmt" -) - -type MarkdownFormatter struct{} - -func (markdownFormatter MarkdownFormatter) MakeBold(text string) string { - return MakeBold(text) -} - -func (markdownFormatter MarkdownFormatter) CreateCodeBlock(title string, content string, language string) string { - return fmt.Sprintf("\n%s\n```%s\n%s\n```\n", markdownFormatter.MakeBold(title), language, content) -} - -func (markdownFormatter MarkdownFormatter) CreateTable(headers []string, rows [][]string) (string, error) { - return CreateTable(headers, rows) -} - -func (markdownFormatter MarkdownFormatter) CreateLink(title string, url string) string { - return CreateLink(title, url) -} - -func (markdownFormatter MarkdownFormatter) CreateHorizontalLine() string { - return CreateHorizontalLine() -} diff --git a/v2/pkg/reporting/exporters/markdown/util/markdown_utils.go b/v2/pkg/reporting/exporters/markdown/util/markdown_utils.go deleted file mode 100644 index b9a6744059..0000000000 --- a/v2/pkg/reporting/exporters/markdown/util/markdown_utils.go +++ /dev/null @@ -1,65 +0,0 @@ -package util - -import ( - "bytes" - "fmt" - "strings" - - errorutil "github.com/projectdiscovery/utils/errors" -) - -func CreateLink(title string, url string) string { - return fmt.Sprintf("[%s](%s)", title, url) -} - -func MakeBold(text string) string { - return "**" + text + "**" -} - -func CreateTable(headers []string, rows [][]string) (string, error) { - builder := &bytes.Buffer{} - headerSize := len(headers) - if headers == nil || headerSize == 0 { - return "", errorutil.New("No headers provided") - } - - builder.WriteString(CreateTableHeader(headers...)) - - for _, row := range rows { - rowSize := len(row) - if rowSize == headerSize { - builder.WriteString(CreateTableRow(row...)) - } else if rowSize < headerSize { - extendedRows := make([]string, headerSize) - copy(extendedRows, row) - builder.WriteString(CreateTableRow(extendedRows...)) - } else { - return "", errorutil.New("Too many columns for the given headers") - } - } - - return builder.String(), nil -} - -func CreateTableHeader(headers ...string) string { - headerSize := len(headers) - if headers == nil || headerSize == 0 { - return "" - } - - return CreateTableRow(headers...) + - "|" + strings.Repeat(" --- |", headerSize) + "\n" -} - -func CreateTableRow(elements ...string) string { - return fmt.Sprintf("| %s |\n", strings.Join(elements, " | ")) -} - -func CreateHeading3(text string) string { - return "### " + text + "\n" -} - -func CreateHorizontalLine() string { - // for regular markdown 3 dashes are enough, but for Jira the minimum is 4 - return "----\n" -} diff --git a/v2/pkg/reporting/exporters/markdown/util/markdown_utils_test.go b/v2/pkg/reporting/exporters/markdown/util/markdown_utils_test.go deleted file mode 100644 index d9d6bc8660..0000000000 --- a/v2/pkg/reporting/exporters/markdown/util/markdown_utils_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package util - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMarkDownHeaderCreation(t *testing.T) { - testCases := []struct { - headers []string - expectedValue string - }{ - {nil, ""}, - {[]string{}, ""}, - {[]string{"one"}, "| one |\n| --- |\n"}, - {[]string{"one", "two"}, "| one | two |\n| --- | --- |\n"}, - {[]string{"one", "two", "three"}, "| one | two | three |\n| --- | --- | --- |\n"}, - } - - for _, currentTestCase := range testCases { - t.Run(strings.Join(currentTestCase.headers, ","), func(t1 *testing.T) { - assert.Equal(t1, CreateTableHeader(currentTestCase.headers...), currentTestCase.expectedValue) - }) - } -} - -func TestCreateTemplateInfoTableTooManyColumns(t *testing.T) { - table, err := CreateTable([]string{"one", "two"}, [][]string{ - {"a", "b", "c"}, - {"d"}, - {"e", "f", "g"}, - {"h", "i"}, - }) - - assert.NotNil(t, err) - assert.Empty(t, table) -} - -func TestCreateTemplateInfoTable1Column(t *testing.T) { - table, err := CreateTable([]string{"one"}, [][]string{{"a"}, {"b"}, {"c"}}) - - expected := `| one | -| --- | -| a | -| b | -| c | -` - - assert.Nil(t, err) - assert.Equal(t, expected, table) -} - -func TestCreateTemplateInfoTable2Columns(t *testing.T) { - table, err := CreateTable([]string{"one", "two"}, [][]string{ - {"a", "b"}, - {"c"}, - {"d", "e"}, - }) - - expected := `| one | two | -| --- | --- | -| a | b | -| c | | -| d | e | -` - - assert.Nil(t, err) - assert.Equal(t, expected, table) -} - -func TestCreateTemplateInfoTable3Columns(t *testing.T) { - table, err := CreateTable([]string{"one", "two", "three"}, [][]string{ - {"a", "b", "c"}, - {"d"}, - {"e", "f", "g"}, - {"h", "i"}, - }) - - expected := `| one | two | three | -| --- | --- | --- | -| a | b | c | -| d | | | -| e | f | g | -| h | i | | -` - - assert.Nil(t, err) - assert.Equal(t, expected, table) -} diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 64e97f8c3c..ca87fec37b 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -60,7 +60,7 @@ func (exporter *Exporter) addToolDetails() { } exporter.sarif.RegisterTool(driver) - reportLocation := sarif.ArtifactLocation{ + reportloc := sarif.ArtifactLocation{ Uri: "file:///" + exporter.options.File, Description: &sarif.Message{ Text: "Nuclei Sarif Report", @@ -70,7 +70,7 @@ func (exporter *Exporter) addToolDetails() { invocation := sarif.Invocation{ CommandLine: os.Args[0], Arguments: os.Args[1:], - ResponseFiles: []sarif.ArtifactLocation{reportLocation}, + ResponseFiles: []sarif.ArtifactLocation{reportloc}, } exporter.sarif.RegisterToolInvocation(invocation) } @@ -102,10 +102,10 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { resultHeader := fmt.Sprintf("%v (%v) found on %v", event.Info.Name, event.TemplateID, event.Host) resultLevel, vulnRating := exporter.getSeverity(severity) - // Extra metadata if generated sarif is uploaded to GitHub security page - ghMeta := map[string]interface{}{} - ghMeta["tags"] = []string{"security"} - ghMeta["security-severity"] = vulnRating + // Extra metdata if generated sarif is uploaded to github security page + ghmeta := map[string]interface{}{} + ghmeta["tags"] = []string{"security"} + ghmeta["security-severity"] = vulnRating // rule contain details of template rule := sarif.ReportingDescriptor{ @@ -115,10 +115,10 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { // Points to template URL Text: event.Info.Description + "\nMore details at\n" + event.TemplateURL + "\n", }, - Properties: ghMeta, + Properties: ghmeta, } - // GitHub Uses ShortDescription as title + // Github Uses ShortDescription as title if event.Info.Description != "" { rule.ShortDescription = &sarif.MultiformatMessageString{ Text: resultHeader, @@ -141,7 +141,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { }, PhysicalLocation: sarif.PhysicalLocation{ ArtifactLocation: sarif.ArtifactLocation{ - // GitHub only accepts file:// protocol and local & relative files only + // github only accepts file:// protocol and local & relative files only // to avoid errors // is used which also translates to file according to specification Uri: "/" + event.Path, Description: &sarif.Message{ @@ -193,4 +193,5 @@ func (exporter *Exporter) Close() error { } return nil + } diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index f801add34a..6e0618dde2 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -1,9 +1,245 @@ package format -type ResultFormatter interface { - MakeBold(text string) string - CreateCodeBlock(title string, content string, language string) string - CreateTable(headers []string, rows [][]string) (string, error) - CreateLink(title string, url string) string - CreateHorizontalLine() string +import ( + "bytes" + "fmt" + "strconv" + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" + + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Summary returns a formatted built one line summary of the event +func Summary(event *output.ResultEvent) string { + template := GetMatchedTemplate(event) + + builder := &strings.Builder{} + builder.WriteString(types.ToString(event.Info.Name)) + builder.WriteString(" (") + builder.WriteString(template) + builder.WriteString(") found on ") + builder.WriteString(event.Host) + data := builder.String() + return data +} + +// MarkdownDescription formats a short description of the generated +// event by the nuclei scanner in Markdown format. +func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the code duplication: format.go <-> jira.go + template := GetMatchedTemplate(event) + builder := &bytes.Buffer{} + builder.WriteString("**Details**: **") + builder.WriteString(template) + builder.WriteString("** ") + + builder.WriteString(" matched at ") + builder.WriteString(event.Host) + + builder.WriteString("\n\n**Protocol**: ") + builder.WriteString(strings.ToUpper(event.Type)) + + builder.WriteString("\n\n**Full URL**: ") + builder.WriteString(event.Matched) + + builder.WriteString("\n\n**Timestamp**: ") + builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) + + builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n") + builder.WriteString(ToMarkdownTableString(&event.Info)) + + if event.Request != "" { + builder.WriteString(createMarkdownCodeBlock("Request", types.ToHexOrString(event.Request), "http")) + } + if event.Response != "" { + var responseString string + // If the response is larger than 5 kb, truncate it before writing. + if len(event.Response) > 5*1024 { + responseString = (event.Response[:5*1024]) + responseString += ".... Truncated ...." + } else { + responseString = event.Response + } + builder.WriteString(createMarkdownCodeBlock("Response", responseString, "http")) + } + + if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { + builder.WriteString("\n**Extra Information**\n\n") + + if len(event.ExtractedResults) > 0 { + builder.WriteString("**Extracted results**:\n\n") + for _, v := range event.ExtractedResults { + builder.WriteString("- ") + builder.WriteString(v) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + if len(event.Metadata) > 0 { + builder.WriteString("**Metadata**:\n\n") + for k, v := range event.Metadata { + builder.WriteString("- ") + builder.WriteString(k) + builder.WriteString(": ") + builder.WriteString(types.ToString(v)) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + } + if event.Interaction != nil { + builder.WriteString("**Interaction Data**\n---\n") + builder.WriteString(event.Interaction.Protocol) + if event.Interaction.QType != "" { + builder.WriteString(" (") + builder.WriteString(event.Interaction.QType) + builder.WriteString(")") + } + builder.WriteString(" Interaction from ") + builder.WriteString(event.Interaction.RemoteAddress) + builder.WriteString(" at ") + builder.WriteString(event.Interaction.UniqueID) + + if event.Interaction.RawRequest != "" { + builder.WriteString(createMarkdownCodeBlock("Interaction Request", event.Interaction.RawRequest, "")) + } + if event.Interaction.RawResponse != "" { + builder.WriteString(createMarkdownCodeBlock("Interaction Response", event.Interaction.RawResponse, "")) + } + } + + reference := event.Info.Reference + if !reference.IsEmpty() { + builder.WriteString("\nReferences: \n") + + referenceSlice := reference.ToSlice() + for i, item := range referenceSlice { + builder.WriteString("- ") + builder.WriteString(item) + if len(referenceSlice)-1 != i { + builder.WriteString("\n") + } + } + } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n**CURL Command**\n```\n") + builder.WriteString(types.ToHexOrString(event.CURLCommand)) + builder.WriteString("\n```") + } + + builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) + data := builder.String() + return data +} + +// GetMatchedTemplate returns the matched template from a result event +func GetMatchedTemplate(event *output.ResultEvent) string { + builder := &strings.Builder{} + builder.WriteString(event.TemplateID) + if event.MatcherName != "" { + builder.WriteString(":") + builder.WriteString(event.MatcherName) + } + if event.ExtractorName != "" { + builder.WriteString(":") + builder.WriteString(event.ExtractorName) + } + template := builder.String() + return template +} + +func ToMarkdownTableString(templateInfo *model.Info) string { + fields := utils.NewEmptyInsertionOrderedStringMap(5) + fields.Set("Name", templateInfo.Name) + fields.Set("Authors", templateInfo.Authors.String()) + fields.Set("Tags", templateInfo.Tags.String()) + fields.Set("Severity", templateInfo.SeverityHolder.Severity.String()) + fields.Set("Description", lineBreakToHTML(templateInfo.Description)) + fields.Set("Remediation", lineBreakToHTML(templateInfo.Remediation)) + + classification := templateInfo.Classification + if classification != nil { + if classification.CVSSMetrics != "" { + generateCVSSMetricsFromClassification(classification, fields) + } + generateCVECWEIDLinksFromClassification(classification, fields) + fields.Set("CVSS-Score", strconv.FormatFloat(classification.CVSSScore, 'f', 2, 64)) + } + + builder := &bytes.Buffer{} + + toMarkDownTable := func(insertionOrderedStringMap *utils.InsertionOrderedStringMap) { + insertionOrderedStringMap.ForEach(func(key string, value interface{}) { + switch value := value.(type) { + case string: + if !utils.IsBlank(value) { + builder.WriteString(fmt.Sprintf("| %s | %s |\n", key, value)) + } + } + }) + } + + toMarkDownTable(fields) + toMarkDownTable(utils.NewInsertionOrderedStringMap(templateInfo.Metadata)) + + return builder.String() +} + +func generateCVSSMetricsFromClassification(classification *model.Classification, fields *utils.InsertionOrderedStringMap) { + // Generate cvss link + var cvssLinkPrefix string + if strings.Contains(classification.CVSSMetrics, "CVSS:3.0") { + cvssLinkPrefix = "https://www.first.org/cvss/calculator/3.0#" + } else if strings.Contains(classification.CVSSMetrics, "CVSS:3.1") { + cvssLinkPrefix = "https://www.first.org/cvss/calculator/3.1#" + } + if cvssLinkPrefix != "" { + fields.Set("CVSS-Metrics", fmt.Sprintf("[%s](%s%s)", classification.CVSSMetrics, cvssLinkPrefix, classification.CVSSMetrics)) + } else { + fields.Set("CVSS-Metrics", classification.CVSSMetrics) + } +} + +func generateCVECWEIDLinksFromClassification(classification *model.Classification, fields *utils.InsertionOrderedStringMap) { + cwes := classification.CWEID.ToSlice() + + cweIDs := make([]string, 0, len(cwes)) + for _, value := range cwes { + parts := strings.Split(value, "-") + if len(parts) != 2 { + continue + } + cweIDs = append(cweIDs, fmt.Sprintf("[%s](https://cwe.mitre.org/data/definitions/%s.html)", strings.ToUpper(value), parts[1])) + } + if len(cweIDs) > 0 { + fields.Set("CWE-ID", strings.Join(cweIDs, ",")) + } + + cves := classification.CVEID.ToSlice() + + cveIDs := make([]string, 0, len(cves)) + for _, value := range cves { + cveIDs = append(cveIDs, fmt.Sprintf("[%s](https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s)", strings.ToUpper(value), value)) + } + if len(cveIDs) > 0 { + fields.Set("CVE-ID", strings.Join(cveIDs, ",")) + } +} + +func createMarkdownCodeBlock(title string, content string, language string) string { + return "\n" + createBoldMarkdown(title) + "\n```" + language + "\n" + content + "\n```\n" +} + +func createBoldMarkdown(value string) string { + return "**" + value + "**" +} + +func lineBreakToHTML(text string) string { + return strings.ReplaceAll(text, "\n", "
") } diff --git a/v2/pkg/reporting/format/format_utils_test.go b/v2/pkg/reporting/format/format_test.go similarity index 86% rename from v2/pkg/reporting/format/format_utils_test.go rename to v2/pkg/reporting/format/format_test.go index 2a950b50b4..6be42bb179 100644 --- a/v2/pkg/reporting/format/format_utils_test.go +++ b/v2/pkg/reporting/format/format_test.go @@ -1,14 +1,14 @@ package format import ( - "github.com/stretchr/testify/assert" "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" ) func TestToMarkdownTableString(t *testing.T) { @@ -25,11 +25,9 @@ func TestToMarkdownTableString(t *testing.T) { }, } - result := CreateTemplateInfoTable(&info, &util.MarkdownFormatter{}) + result := ToMarkdownTableString(&info) - expectedOrderedAttributes := `| Key | Value | -| --- | --- | -| Name | Test Template Name | + expectedOrderedAttributes := `| Name | Test Template Name | | Authors | forgedhallpass, ice3man | | Tags | cve, misc | | Severity | high | diff --git a/v2/pkg/reporting/format/format_utils.go b/v2/pkg/reporting/format/format_utils.go deleted file mode 100644 index c02f00622d..0000000000 --- a/v2/pkg/reporting/format/format_utils.go +++ /dev/null @@ -1,229 +0,0 @@ -package format - -import ( - "bytes" - "fmt" - "strconv" - "strings" - - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v2/pkg/model" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" -) - -// Summary returns a formatted built one line summary of the event -func Summary(event *output.ResultEvent) string { - return fmt.Sprintf("%s (%s) found on %s", types.ToString(event.Info.Name), GetMatchedTemplateName(event), event.Host) -} - -// GetMatchedTemplateName returns the matched template name from a result event -// together with the found matcher and extractor name, if present -func GetMatchedTemplateName(event *output.ResultEvent) string { - matchedTemplateName := event.TemplateID - if event.MatcherName != "" { - matchedTemplateName += ":" + event.MatcherName - } - - if event.ExtractorName != "" { - matchedTemplateName += ":" + event.ExtractorName - } - - return matchedTemplateName -} - -func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter) string { - template := GetMatchedTemplateName(event) - builder := &bytes.Buffer{} - builder.WriteString(fmt.Sprintf("%s: %s matched at %s\n\n", formatter.MakeBold("Details"), formatter.MakeBold(template), event.Host)) - - attributes := utils.NewEmptyInsertionOrderedStringMap(3) - attributes.Set("Protocol", strings.ToUpper(event.Type)) - attributes.Set("Full URL", event.Matched) - attributes.Set("Timestamp", event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) - attributes.ForEach(func(key string, data interface{}) { - builder.WriteString(fmt.Sprintf("%s: %s\n\n", formatter.MakeBold(key), types.ToString(data))) - }) - - builder.WriteString(formatter.MakeBold("Template Information")) - builder.WriteString("\n\n") - builder.WriteString(CreateTemplateInfoTable(&event.Info, formatter)) - - if event.Request != "" { - builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http")) - } - if event.Response != "" { - var responseString string - // If the response is larger than 5 kb, truncate it before writing. - maxKbSize := 5 * 1024 - if len(event.Response) > maxKbSize { - responseString = event.Response[:maxKbSize] - responseString += ".... Truncated ...." - } else { - responseString = event.Response - } - builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http")) - } - - if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { - builder.WriteString("\n") - builder.WriteString(formatter.MakeBold("Extra Information")) - builder.WriteString("\n\n") - - if len(event.ExtractedResults) > 0 { - builder.WriteString(formatter.MakeBold("Extracted results:")) - builder.WriteString("\n\n") - - for _, v := range event.ExtractedResults { - builder.WriteString("- ") - builder.WriteString(v) - builder.WriteString("\n") - } - builder.WriteString("\n") - } - if len(event.Metadata) > 0 { - builder.WriteString(formatter.MakeBold("Metadata:")) - builder.WriteString("\n\n") - for k, v := range event.Metadata { - builder.WriteString("- ") - builder.WriteString(k) - builder.WriteString(": ") - builder.WriteString(types.ToString(v)) - builder.WriteString("\n") - } - builder.WriteString("\n") - } - } - if event.Interaction != nil { - builder.WriteString(fmt.Sprintf("%s\n%s", formatter.MakeBold("Interaction Data"), formatter.CreateHorizontalLine())) - builder.WriteString(event.Interaction.Protocol) - if event.Interaction.QType != "" { - builder.WriteString(fmt.Sprintf(" (%s)", event.Interaction.QType)) - } - builder.WriteString(fmt.Sprintf(" Interaction from %s at %s", event.Interaction.RemoteAddress, event.Interaction.UniqueID)) - - if event.Interaction.RawRequest != "" { - builder.WriteString(formatter.CreateCodeBlock("Interaction Request", event.Interaction.RawRequest, "")) - } - if event.Interaction.RawResponse != "" { - builder.WriteString(formatter.CreateCodeBlock("Interaction Response", event.Interaction.RawResponse, "")) - } - } - - reference := event.Info.Reference - if !reference.IsEmpty() { - builder.WriteString("\nReferences: \n") - - referenceSlice := reference.ToSlice() - for i, item := range referenceSlice { - builder.WriteString("- ") - builder.WriteString(item) - if len(referenceSlice)-1 != i { - builder.WriteString("\n") - } - } - } - builder.WriteString("\n") - - if event.CURLCommand != "" { - builder.WriteString( - formatter.CreateCodeBlock("CURL command", types.ToHexOrString(event.CURLCommand), "sh"), - ) - } - - builder.WriteString("\n" + formatter.CreateHorizontalLine() + "\n") - builder.WriteString(fmt.Sprintf("Generated by %s", formatter.CreateLink("Nuclei "+config.Version, "https://github.com/projectdiscovery/nuclei"))) - data := builder.String() - return data -} - -func CreateTemplateInfoTable(templateInfo *model.Info, formatter ResultFormatter) string { - rows := [][]string{ - {"Name", templateInfo.Name}, - {"Authors", templateInfo.Authors.String()}, - {"Tags", templateInfo.Tags.String()}, - {"Severity", templateInfo.SeverityHolder.Severity.String()}, - } - - if !utils.IsBlank(templateInfo.Description) { - rows = append(rows, []string{"Description", lineBreakToHTML(templateInfo.Description)}) - } - - if !utils.IsBlank(templateInfo.Remediation) { - rows = append(rows, []string{"Remediation", lineBreakToHTML(templateInfo.Remediation)}) - } - - classification := templateInfo.Classification - if classification != nil { - if classification.CVSSMetrics != "" { - rows = append(rows, []string{"CVSS-Metrics", generateCVSSMetricsFromClassification(classification)}) - } - - rows = append(rows, generateCVECWEIDLinksFromClassification(classification)...) - rows = append(rows, []string{"CVSS-Score", strconv.FormatFloat(classification.CVSSScore, 'f', 2, 64)}) - } - - for key, value := range templateInfo.Metadata { - switch value := value.(type) { - case string: - if !utils.IsBlank(value) { - rows = append(rows, []string{key, value}) - } - } - } - - table, _ := formatter.CreateTable([]string{"Key", "Value"}, rows) - - return table -} - -func generateCVSSMetricsFromClassification(classification *model.Classification) string { - var cvssLinkPrefix string - if strings.Contains(classification.CVSSMetrics, "CVSS:3.0") { - cvssLinkPrefix = "https://www.first.org/cvss/calculator/3.0#" - } else if strings.Contains(classification.CVSSMetrics, "CVSS:3.1") { - cvssLinkPrefix = "https://www.first.org/cvss/calculator/3.1#" - } - - if cvssLinkPrefix == "" { - return classification.CVSSMetrics - } else { - return util.CreateLink(classification.CVSSMetrics, cvssLinkPrefix+classification.CVSSMetrics) - } -} - -func generateCVECWEIDLinksFromClassification(classification *model.Classification) [][]string { - cwes := classification.CWEID.ToSlice() - - cweIDs := make([]string, 0, len(cwes)) - for _, value := range cwes { - parts := strings.Split(value, "-") - if len(parts) != 2 { - continue - } - cweIDs = append(cweIDs, util.CreateLink(strings.ToUpper(value), fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", parts[1]))) - } - - var rows [][]string - - if len(cweIDs) > 0 { - rows = append(rows, []string{"CWE-ID", strings.Join(cweIDs, ",")}) - } - - cves := classification.CVEID.ToSlice() - cveIDs := make([]string, 0, len(cves)) - for _, value := range cves { - cveIDs = append(cveIDs, util.CreateLink(strings.ToUpper(value), fmt.Sprintf("https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s", value))) - } - if len(cveIDs) > 0 { - rows = append(rows, []string{"CVE-ID", strings.Join(cveIDs, ",")}) - } - - return rows -} - -func lineBreakToHTML(text string) string { - return strings.ReplaceAll(text, "\n", "
") -} diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 4def0d2fe3..086f9c7ed5 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -7,12 +7,12 @@ import ( "net/url" "strings" + "golang.org/x/oauth2" + "github.com/google/go-github/github" "github.com/pkg/errors" - "golang.org/x/oauth2" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/retryablehttp-go" @@ -28,7 +28,7 @@ type Integration struct { type Options struct { // BaseURL (optional) is the self-hosted GitHub application url BaseURL string `yaml:"base-url" validate:"omitempty,url"` - // Username is the username of the GitHub user + // Username is the username of the github user Username string `yaml:"username" validate:"required"` // Owner is the owner name of the repository for issues. Owner string `yaml:"owner" validate:"required"` @@ -78,7 +78,7 @@ func New(options *Options) (*Integration, error) { // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) - description := format.CreateReportDescription(event, util.MarkdownFormatter{}) + description := format.MarkdownDescription(event) labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) if i.options.SeverityAsLabel && severityLabel != "" { diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index b83b4052d5..68bd477bb1 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -6,7 +6,6 @@ import ( "github.com/xanzy/go-gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/projectdiscovery/retryablehttp-go" ) @@ -60,7 +59,7 @@ func New(options *Options) (*Integration, error) { // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) - description := format.CreateReportDescription(event, util.MarkdownFormatter{}) + description := format.MarkdownDescription(event) labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) if i.options.SeverityAsLabel && severityLabel != "" { diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index af8853c755..3b5fdad798 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -1,6 +1,7 @@ package jira import ( + "bytes" "fmt" "io" "strings" @@ -9,41 +10,15 @@ import ( "github.com/trivago/tgo/tcontainer" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/retryablehttp-go" ) -type Formatter struct { - util.MarkdownFormatter -} - -func (jiraFormatter *Formatter) MakeBold(text string) string { - return "*" + text + "*" -} - -func (jiraFormatter *Formatter) CreateCodeBlock(title string, content string, _ string) string { - return fmt.Sprintf("\n%s\n{code}\n%s\n{code}\n", jiraFormatter.MakeBold(title), content) -} - -func (jiraFormatter *Formatter) CreateTable(headers []string, rows [][]string) (string, error) { - table, err := jiraFormatter.MarkdownFormatter.CreateTable(headers, rows) - if err != nil { - return "", err - } - tableRows := strings.Split(table, "\n") - tableRowsWithoutHeaderSeparator := append(tableRows[:1], tableRows[2:]...) - return strings.Join(tableRowsWithoutHeaderSeparator, "\n"), nil -} - -func (jiraFormatter *Formatter) CreateLink(title string, url string) string { - return fmt.Sprintf("[%s|%s]", title, url) -} - // Integration is a client for an issue tracker integration type Integration struct { - Formatter jira *jira.Client options *Options } @@ -154,7 +129,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { } } fields := &jira.IssueFields{ - Description: format.CreateReportDescription(event, i), + Description: jiraFormatDescription(event), Unknowns: customFields, Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, @@ -164,7 +139,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { if !i.options.Cloud { fields = &jira.IssueFields{ Assignee: &jira.User{Name: i.options.AccountID}, - Description: format.CreateReportDescription(event, i), + Description: jiraFormatDescription(event), Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, @@ -196,7 +171,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error { return err } else if issueID != "" { _, _, err = i.jira.Issue.AddComment(issueID, &jira.Comment{ - Body: format.CreateReportDescription(event, i), + Body: jiraFormatDescription(event), }) return err } @@ -206,7 +181,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error { // FindExistingIssue checks if the issue already exists and returns its ID func (i *Integration) FindExistingIssue(event *output.ResultEvent) (string, error) { - template := format.GetMatchedTemplateName(event) + template := format.GetMatchedTemplate(event) jql := fmt.Sprintf("summary ~ \"%s\" AND summary ~ \"%s\" AND status != \"%s\"", template, event.Host, i.options.StatusNot) searchOptions := &jira.SearchOptions{ @@ -233,3 +208,117 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (string, erro return chunk[0].ID, nil } } + +// jiraFormatDescription formats a short description of the generated +// event by the nuclei scanner in Jira format. +func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove the code duplication: format.go <-> jira.go + template := format.GetMatchedTemplate(event) + + builder := &bytes.Buffer{} + builder.WriteString("*Details*: *") + builder.WriteString(template) + builder.WriteString("* ") + + builder.WriteString(" matched at ") + builder.WriteString(event.Host) + + builder.WriteString("\n\n*Protocol*: ") + builder.WriteString(strings.ToUpper(event.Type)) + + builder.WriteString("\n\n*Full URL*: ") + builder.WriteString(event.Matched) + + builder.WriteString("\n\n*Timestamp*: ") + builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) + + builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n") + builder.WriteString(format.ToMarkdownTableString(&event.Info)) + + builder.WriteString(createMarkdownCodeBlock("Request", event.Request)) + + builder.WriteString("\n*Response*\n\n{code}\n") + // If the response is larger than 5 kb, truncate it before writing. + if len(event.Response) > 5*1024 { + builder.WriteString(event.Response[:5*1024]) + builder.WriteString(".... Truncated ....") + } else { + builder.WriteString(event.Response) + } + builder.WriteString("\n{code}\n\n") + + if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { + builder.WriteString("\n*Extra Information*\n\n") + if len(event.ExtractedResults) > 0 { + builder.WriteString("*Extracted results*:\n\n") + for _, v := range event.ExtractedResults { + builder.WriteString("- ") + builder.WriteString(v) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + if len(event.Metadata) > 0 { + builder.WriteString("*Metadata*:\n\n") + for k, v := range event.Metadata { + builder.WriteString("- ") + builder.WriteString(k) + builder.WriteString(": ") + builder.WriteString(types.ToString(v)) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + } + if event.Interaction != nil { + builder.WriteString("*Interaction Data*\n---\n") + builder.WriteString(event.Interaction.Protocol) + if event.Interaction.QType != "" { + builder.WriteString(" (") + builder.WriteString(event.Interaction.QType) + builder.WriteString(")") + } + builder.WriteString(" Interaction from ") + builder.WriteString(event.Interaction.RemoteAddress) + builder.WriteString(" at ") + builder.WriteString(event.Interaction.UniqueID) + + if event.Interaction.RawRequest != "" { + builder.WriteString(createMarkdownCodeBlock("Interaction Request", event.Interaction.RawRequest)) + } + if event.Interaction.RawResponse != "" { + builder.WriteString(createMarkdownCodeBlock("Interaction Response", event.Interaction.RawResponse)) + } + } + + reference := event.Info.Reference + if !reference.IsEmpty() { + builder.WriteString("\nReferences: \n") + + referenceSlice := reference.ToSlice() + for i, item := range referenceSlice { + builder.WriteString("- ") + builder.WriteString(item) + if len(referenceSlice)-1 != i { + builder.WriteString("\n") + } + } + } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n*CURL Command*\n{code}\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n{code}") + } + builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) + data := builder.String() + return data +} + +func createMarkdownCodeBlock(title string, content string) string { + return "\n" + createBoldMarkdown(title) + "\n" + content + "*\n\n{code}" +} + +func createBoldMarkdown(value string) string { + return "*" + value + "*\n\n{code}" +} diff --git a/v2/pkg/reporting/trackers/jira/jira_test.go b/v2/pkg/reporting/trackers/jira/jira_test.go deleted file mode 100644 index 6a9e9a31df..0000000000 --- a/v2/pkg/reporting/trackers/jira/jira_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package jira - -import ( - "github.com/stretchr/testify/assert" - "strings" - "testing" -) - -func TestLinkCreation(t *testing.T) { - jiraIntegration := &Integration{} - link := jiraIntegration.CreateLink("ProjectDiscovery", "https://projectdiscovery.io") - assert.Equal(t, "[ProjectDiscovery|https://projectdiscovery.io]", link) -} - -func TestHorizontalLineCreation(t *testing.T) { - jiraIntegration := &Integration{} - horizontalLine := jiraIntegration.CreateHorizontalLine() - assert.True(t, strings.Contains(horizontalLine, "----")) -} - -func TestTableCreation(t *testing.T) { - jiraIntegration := &Integration{} - - table, err := jiraIntegration.CreateTable([]string{"key", "value"}, [][]string{ - {"a", "b"}, - {"c"}, - {"d", "e"}, - }) - - assert.Nil(t, err) - expected := `| key | value | -| a | b | -| c | | -| d | e | -` - assert.Equal(t, expected, table) -} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 101e7c1bdc..4ce2731b73 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -360,8 +360,6 @@ type Options struct { FuzzingType string // Fuzzing Mode overrides template level fuzzing-mode configuration FuzzingMode string - // TlsImpersonate enables TLS impersonation - TlsImpersonate bool } // ShouldLoadResume resume file