From a79a9911cceb505117356be81f72d1aadc7325c9 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Mon, 15 Jan 2024 09:52:39 +0100 Subject: [PATCH 01/10] Add vendor for SEG detection : Mailinblack --- engines/owl_dns/Dockerfile | 6 +++--- engines/owl_dns/VERSION | 2 +- engines/owl_dns/__init__.py | 10 +++++----- engines/owl_dns/etc/seg_list.json | 10 ++++++++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index e6c97fde..7a5e85aa 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,10 +1,10 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.6" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7" # Install dependencies RUN apk add --update --no-cache \ - python3 python3-dev py3-pip \ - git \ + python3 python3-dev py3-pip \ + git \ && rm -rf /var/cache/apk/* # Create the target repo diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index eac1e0ad..f01291b8 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.6 +1.5.7 diff --git a/engines/owl_dns/__init__.py b/engines/owl_dns/__init__.py index da4cdc82..8ca87e46 100644 --- a/engines/owl_dns/__init__.py +++ b/engines/owl_dns/__init__.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__title__ = 'patrowl_engine_owl_dns' -__version__ = '1.5.5' -__author__ = 'Nicolas MATTIOCCO' -__license__ = 'AGPLv3' -__copyright__ = 'Copyright (C) 2018-2023 Nicolas Mattiocco - @MaKyOtOx' +__title__ = "patrowl_engine_owl_dns" +__version__ = "1.5.7" +__author__ = "Nicolas MATTIOCCO" +__license__ = "AGPLv3" +__copyright__ = "Copyright (C) 2018-2024 Nicolas Mattiocco - @MaKyOtOx" diff --git a/engines/owl_dns/etc/seg_list.json b/engines/owl_dns/etc/seg_list.json index 5eef5c8e..5a17fc2c 100644 --- a/engines/owl_dns/etc/seg_list.json +++ b/engines/owl_dns/etc/seg_list.json @@ -89,6 +89,16 @@ "links": [ "https://success.trendmicro.com/dcx/s/solution/1055888-redirecting-mail-exchange-mx-records-to-hosted-email-security-hes?language=en_US&sfdcIFrameOrigin=null" ] + }, + "mailinblack": { + "provider": "mailinblack", + "product": "Mailinblack", + "mx_records": [ + ".mailinblack.com." + ], + "links": [ + "https://support.mailinblack.com/fr/articles/6853774-online-comment-editer-les-parametres-dns-mx-d-un-domaine" + ] } } } \ No newline at end of file From d04ccc6b4ead2d913fc268da9c00e337cfecf3c1 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Mon, 15 Jan 2024 09:54:40 +0100 Subject: [PATCH 02/10] ARS-272 Add vendor for SEG detection : Mailinblack --- AUTHORS | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 773084b6..c6c6c138 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,4 +7,4 @@ Crontributors ------- * CERT Banque de France (CERT-BDF) -Copyright (C) 2018-2022 Nicolas MATTIOCCO +Copyright (C) 2018-2024 Nicolas MATTIOCCO diff --git a/VERSION b/VERSION index 83d4cf8a..e7a15ceb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.18 +1.5.19 From c26b02662b8cde083df2416444e44ce6f4feb615 Mon Sep 17 00:00:00 2001 From: sebastien Date: Tue, 16 Jan 2024 16:10:33 +0100 Subject: [PATCH 03/10] Updated VERSION --- VERSION | 2 +- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- engines/owl_dns/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index e7a15ceb..5ebccafe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.19 +1.5.19-rc1 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index 7a5e85aa..c52f95a3 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7-rc1" # Install dependencies RUN apk add --update --no-cache \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index f01291b8..b2ef0857 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.7 +1.5.7-rc1 diff --git a/engines/owl_dns/__init__.py b/engines/owl_dns/__init__.py index 8ca87e46..1950764b 100644 --- a/engines/owl_dns/__init__.py +++ b/engines/owl_dns/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __title__ = "patrowl_engine_owl_dns" -__version__ = "1.5.7" +__version__ = "1.5.7-rc1" __author__ = "Nicolas MATTIOCCO" __license__ = "AGPLv3" __copyright__ = "Copyright (C) 2018-2024 Nicolas Mattiocco - @MaKyOtOx" From c6477fda7ca1fe0ca81c6bf00318bb5996c2f2d1 Mon Sep 17 00:00:00 2001 From: sebastien Date: Tue, 16 Jan 2024 16:14:02 +0100 Subject: [PATCH 04/10] Updated VERSION --- engines/owl_dns/owl_dns.json.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/owl_dns/owl_dns.json.sample b/engines/owl_dns/owl_dns.json.sample index f2eebd8e..f5873fdb 100644 --- a/engines/owl_dns/owl_dns.json.sample +++ b/engines/owl_dns/owl_dns.json.sample @@ -1,6 +1,6 @@ { "name": "PatrOwl - Dns module", - "version": "0.1", + "version": "1.5.7-rc1", "description": "DNS Scanner", "allowed_asset_types": ["ip", "domain", "fqdn", "keyword"], "sublist3r_bin_path": "/opt/patrowl-engines/owl_dns/external-libs/Sublist3r", From 0f56f71a047c05276afbbff6828b37e663a681a1 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Thu, 18 Jan 2024 17:00:42 +0100 Subject: [PATCH 05/10] ARS-285: NMAP fix error 'issues not available yet' on getfindings --- engines/nmap/engine-nmap.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/engines/nmap/engine-nmap.py b/engines/nmap/engine-nmap.py index e4df5010..7512877c 100644 --- a/engines/nmap/engine-nmap.py +++ b/engines/nmap/engine-nmap.py @@ -135,7 +135,7 @@ def start(): ) return jsonify(res), 503 - if type(data["options"]) == str: + if type(data["options"]) is str: data["options"] = json.loads(data["options"]) scan = { @@ -278,7 +278,7 @@ def _scan_thread(scan_id): cmd_sec = split(cmd) this.scans[scan_id]["proc_cmd"] = "not set!!" - with open(log_path, "w") as stderr: + with open(log_path, "w"): this.scans[scan_id]["proc"] = subprocess.Popen( cmd_sec, shell=False, @@ -335,11 +335,15 @@ def _scan_thread(scan_id): # print(f'scan {scan_id} is finished !') break + time.sleep(1) # wait for creating report file (could be long) + # Check if the report is available (exists && scan finished) report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml" if not os.path.exists(report_filename): - this.scans[scan_id]["status"] = "FINISHED" # ERROR ? - this.scans[scan_id]["issues_available"] = True + # this.scans[scan_id]["status"] = "FINISHED" # ERROR ? + # this.scans[scan_id]["issues_available"] = True + this.scans[scan_id]["status"] = "ERROR" + this.scans[scan_id]["issues_available"] = False return False try: @@ -466,6 +470,14 @@ def scan_status(scan_id): this.scans[scan_id]["status"] = "FINISHED" # print(f"scan_status/scan '{scan_id}' is finished") + elif ( + not psutil.pid_exists(proc.pid) + and this.scans[scan_id]["issues_available"] is False + and this.scans[scan_id]["status"] == "ERROR" + ): + res.update({"status": "ERROR"}) + # print(f"scan_status/scan '{scan_id}' is finished") + elif psutil.pid_exists(proc.pid) and psutil.Process(proc.pid).status() in [ "sleeping", "running", @@ -1101,7 +1113,7 @@ def _parse_report(filename, scan_id): def _get_cpe_link(cpe): - return "https://nvd.nist.gov/vuln/search/results?adv_search=true&cpe={}".format(cpe) + return f"https://nvd.nist.gov/vuln/search/results?adv_search=true&cpe={cpe}" # custom functions for Vulners issues @@ -1148,7 +1160,7 @@ def getfindings(scan_id): return jsonify(res) # check if the report is available (exists && scan finished) - report_filename = BASE_DIR + "/results/nmap_{}.xml".format(scan_id) + report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml" if not os.path.exists(report_filename): res.update({"status": "error", "reason": "Report file not available"}) return jsonify(res) @@ -1240,7 +1252,7 @@ def page_not_found(e): @app.before_first_request def main(): - #if os.getuid() != 0: #run with root because of docker env vars scope + # if os.getuid() != 0: #run with root because of docker env vars scope # app.logger.error("Start the NMAP engine using root privileges !") # sys.exit(-1) if not os.path.exists(f"{BASE_DIR}/results"): From d497cd331f6e6b136a48dd19d51cb816db3a3de8 Mon Sep 17 00:00:00 2001 From: sebastien Date: Thu, 18 Jan 2024 17:59:26 +0100 Subject: [PATCH 06/10] Updated VERSION --- engines/nmap/Dockerfile | 2 +- engines/nmap/VERSION | 2 +- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- engines/owl_dns/__init__.py | 2 +- engines/owl_dns/owl_dns.json.sample | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/engines/nmap/Dockerfile b/engines/nmap/Dockerfile index c674327b..56336cd1 100644 --- a/engines/nmap/Dockerfile +++ b/engines/nmap/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.45" +LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.46rc1" # Set the working directory RUN mkdir -p /opt/patrowl-engines/nmap diff --git a/engines/nmap/VERSION b/engines/nmap/VERSION index db946b7c..d03f39c9 100644 --- a/engines/nmap/VERSION +++ b/engines/nmap/VERSION @@ -1 +1 @@ -1.4.45 +1.4.46rc1 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index c52f95a3..ffff34d8 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7-rc1" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7-rc2" # Install dependencies RUN apk add --update --no-cache \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index b2ef0857..2cc24f0b 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.7-rc1 +1.5.7-rc2 diff --git a/engines/owl_dns/__init__.py b/engines/owl_dns/__init__.py index 1950764b..1d4fbc03 100644 --- a/engines/owl_dns/__init__.py +++ b/engines/owl_dns/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __title__ = "patrowl_engine_owl_dns" -__version__ = "1.5.7-rc1" +__version__ = "1.5.7-rc2" __author__ = "Nicolas MATTIOCCO" __license__ = "AGPLv3" __copyright__ = "Copyright (C) 2018-2024 Nicolas Mattiocco - @MaKyOtOx" diff --git a/engines/owl_dns/owl_dns.json.sample b/engines/owl_dns/owl_dns.json.sample index f5873fdb..59407987 100644 --- a/engines/owl_dns/owl_dns.json.sample +++ b/engines/owl_dns/owl_dns.json.sample @@ -1,6 +1,6 @@ { "name": "PatrOwl - Dns module", - "version": "1.5.7-rc1", + "version": "1.5.7-rc2", "description": "DNS Scanner", "allowed_asset_types": ["ip", "domain", "fqdn", "keyword"], "sublist3r_bin_path": "/opt/patrowl-engines/owl_dns/external-libs/Sublist3r", From 77dfdf6e868b5ed67134d9de93d9018d5f8863fb Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Thu, 18 Jan 2024 21:55:30 +0100 Subject: [PATCH 07/10] ARS-285 fix cpe parsing [nvnc] --- engines/nmap/engine-nmap.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/engines/nmap/engine-nmap.py b/engines/nmap/engine-nmap.py index 7512877c..a4841d0a 100644 --- a/engines/nmap/engine-nmap.py +++ b/engines/nmap/engine-nmap.py @@ -3,6 +3,7 @@ import os import subprocess import sys +import traceback import psutil import json import optparse @@ -363,8 +364,12 @@ def _scan_thread(scan_id): issues.extend(extra_issues) this.scans[scan_id]["issues"] = deepcopy(issues) - except Exception: - pass + except Exception as e: + print(e) + app.logger.info(e) + traceback.print_exception(*sys.exc_info()) + this.scans[scan_id]["status"] = "ERROR" + this.scans[scan_id]["issues_available"] = False this.scans[scan_id]["issues_available"] = True this.scans[scan_id]["status"] = "FINISHED" @@ -437,7 +442,7 @@ def stop_scan(scan_id): ) this.scans[scan_id]["status"] = "STOPPED" - this.scans[scan_id]["finished_at"] = int(time.time() * 1000) + # this.scans[scan_id]["finished_at"] = int(time.time() * 1000) return jsonify(res) @@ -746,7 +751,9 @@ def _parse_report(filename, scan_id): os_data["name"] = osinfo.get("name") os_data["accuracy"] = osinfo.get("accuracy") for osclass in osinfo.findall("osclass"): - os_data["cpe"].append(osclass.find("cpe").text) + os_cpe = osclass.find("cpe") + if os_cpe is not None: + os_data["cpe"].append(os_cpe.text) res.append( deepcopy( _add_issue( From 9ef2d9f52f9d254af40ea96a3cb26f9a4cc79a99 Mon Sep 17 00:00:00 2001 From: sebastien Date: Fri, 19 Jan 2024 11:02:00 +0100 Subject: [PATCH 08/10] Updated VERSION (nmap) --- engines/nmap/Dockerfile | 2 +- engines/nmap/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engines/nmap/Dockerfile b/engines/nmap/Dockerfile index 56336cd1..96dcd19e 100644 --- a/engines/nmap/Dockerfile +++ b/engines/nmap/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.46rc1" +LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.46" # Set the working directory RUN mkdir -p /opt/patrowl-engines/nmap diff --git a/engines/nmap/VERSION b/engines/nmap/VERSION index d03f39c9..7c3125c0 100644 --- a/engines/nmap/VERSION +++ b/engines/nmap/VERSION @@ -1 +1 @@ -1.4.46rc1 +1.4.46 From dd62b91ca7277812650fe3cb55b68cb2b9f01d2c Mon Sep 17 00:00:00 2001 From: sebastien Date: Fri, 19 Jan 2024 11:02:18 +0100 Subject: [PATCH 09/10] Updated VERSION (owl_dns) --- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- engines/owl_dns/__init__.py | 2 +- engines/owl_dns/owl_dns.json.sample | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index ffff34d8..7a5e85aa 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7-rc2" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.7" # Install dependencies RUN apk add --update --no-cache \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index 2cc24f0b..f01291b8 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.7-rc2 +1.5.7 diff --git a/engines/owl_dns/__init__.py b/engines/owl_dns/__init__.py index 1d4fbc03..8ca87e46 100644 --- a/engines/owl_dns/__init__.py +++ b/engines/owl_dns/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __title__ = "patrowl_engine_owl_dns" -__version__ = "1.5.7-rc2" +__version__ = "1.5.7" __author__ = "Nicolas MATTIOCCO" __license__ = "AGPLv3" __copyright__ = "Copyright (C) 2018-2024 Nicolas Mattiocco - @MaKyOtOx" diff --git a/engines/owl_dns/owl_dns.json.sample b/engines/owl_dns/owl_dns.json.sample index 59407987..e0a651b4 100644 --- a/engines/owl_dns/owl_dns.json.sample +++ b/engines/owl_dns/owl_dns.json.sample @@ -1,6 +1,6 @@ { "name": "PatrOwl - Dns module", - "version": "1.5.7-rc2", + "version": "1.5.7", "description": "DNS Scanner", "allowed_asset_types": ["ip", "domain", "fqdn", "keyword"], "sublist3r_bin_path": "/opt/patrowl-engines/owl_dns/external-libs/Sublist3r", From 6e39853aeeec4ba23a5a6d713143b42e2b93a78d Mon Sep 17 00:00:00 2001 From: sebastien Date: Fri, 19 Jan 2024 11:02:30 +0100 Subject: [PATCH 10/10] 1.5.19 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5ebccafe..e7a15ceb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.19-rc1 +1.5.19