Merge branch 'features/redis-reporter' into alpha

megalinter/Linter.py

@@ -138,6 +138,7 @@ def __init__(self, params=None, linter_config=None):
 
         self.report_folder = ""
         self.reporters = []
+        self.lint_command_log: list(str) | str | None = None
 
         # Initialize parameters
         default_params = {
@@ -891,7 +892,11 @@ def process_linter(self, file=None):
             os.remove(self.sarif_output_file)
         # Build command using method locally defined on Linter class
         command = self.build_lint_command(file)
-        logging.debug(f"[{self.linter_name}] command: {str(command)}")
+        # Output command if debug mode
+        if os.environ.get("LOG_LEVEL", "INFO") == "DEBUG":
+            logging.debug(f"[{self.linter_name}] command: {str(command)}")
+            self.lint_command_log = command
+        # Run command via CLI
         return_code, return_output = self.execute_lint_command(command)
         logging.debug(
             f"[{self.linter_name}] result: {str(return_code)} {return_output}"

megalinter/reporters/RedisLinterReporter.py

@@ -45,7 +45,7 @@ def manage_activation(self):
                     )
                 )
                 self.redis_method = config.get(
-                    self.master.request_id, "REDIS_REPORTER_METHOD", "STREAM"
+                    self.master.request_id, "REDIS_REPORTER_METHOD", "PUBSUB"
                 )
                 # Use Redis Stream
                 if self.redis_method == "STREAM":
@@ -59,7 +59,7 @@ def manage_activation(self):
                     self.pubsub_channel = config.get(
                         self.master.request_id,
                         "REDIS_LINTER_REPORTER_PUBSUB_CHANNEL",
-                        "megalinter:pubsub:linter_results:" + self.master.request_id,
+                        "megalinter:pubsub:" + self.master.request_id,
                     )
             else:
                 logging.error(

megalinter/reporters/RedisReporter.py

@@ -43,7 +43,7 @@ def manage_activation(self):
                     )
                 )
                 self.redis_method = config.get(
-                    self.master.request_id, "REDIS_REPORTER_METHOD", "STREAM"
+                    self.master.request_id, "REDIS_REPORTER_METHOD", "PUBSUB"
                 )
                 # Use Redis Stream
                 if self.redis_method == "STREAM":
@@ -57,7 +57,7 @@ def manage_activation(self):
                     self.pubsub_channel = config.get(
                         self.master.request_id,
                         "REDIS_REPORTER_PUBSUB_CHANNEL",
-                        "megalinter:pubsub:results:" + self.master.request_id,
+                        "megalinter:pubsub:" + self.master.request_id,
                     )
             else:
                 logging.error(

megalinter/utils_reporter.py

@@ -227,6 +227,7 @@ def convert_sarif_to_human(sarif_in, request_id) -> str:
 
 def build_reporter_start_message(reporter, redis_stream=False) -> dict:
     result = {
+        "messageType": "megalinterStart",
         "megaLinterStatus": "created",
         "linters": [],
         "requestId": reporter.master.request_id,
@@ -240,12 +241,16 @@ def build_reporter_start_message(reporter, redis_stream=False) -> dict:
 
 
 def build_reporter_external_result(reporter, redis_stream=False) -> dict:
-    result = {"megaLinterStatus": "completed", "requestId": reporter.master.request_id}
+    result = {
+        "messageType": "megalinterComplete",
+        "megaLinterStatus": "completed",
+        "requestId": reporter.master.request_id,
+    }
     return manage_redis_stream(result, redis_stream)
 
 
 def build_linter_reporter_start_message(reporter, redis_stream=False) -> dict:
-    result = {"linterStatus": "started"}
+    result = {"messageType": "linterStart", "linterStatus": "started"}
     result = result | get_linter_infos(reporter.master)
     return manage_redis_stream(result, redis_stream)
 
@@ -262,11 +267,14 @@ def build_linter_reporter_external_result(reporter, redis_stream=False) -> dict:
         else error_msg
     )
     result = {
+        "messageType": "linterComplete",
         "linterStatus": "success" if reporter.master.return_code == 0 else "error",
         "linterErrorNumber": reporter.master.total_number_errors,
         "linterStatusMessage": status_message,
         "linterElapsedTime": round(reporter.master.elapsed_time_s, 2),
     }
+    if reporter.master.lint_command_log is not None:
+        result["linterCliCommand"] = reporter.master.lint_command_log
     result = result | get_linter_infos(reporter.master)
     if (
         reporter.master.sarif_output_file is not None

server/docker-compose-dev.yml

@@ -35,7 +35,7 @@ services:
     environment:
       # Worker variables
       - MEGALINTER_SERVER=true
-
+      - LOG_LEVEL=DEBUG    # Much more logs including run commands
       # Redis Reporter info: can be different than Redis job Queue server
       - REDIS_REPORTER=true
       - REDIS_REPORTER_HOST=megalinter_server_redis
@@ -47,16 +47,17 @@ services:
       #- REDIS_LINTER_REPORTER_METHOD=STREAM
       #- REDIS_LINTER_REPORTER_STREAM=megalinter:stream:linter_results
 
-      # PubSub mode
-      - REDIS_REPORTER_METHOD=PUBSUB
-      - REDIS_LINTER_REPORTER_METHOD=PUBSUB
-      ## You can force a channel. If not set, megalinter:pubsub:results:REQUEST_ID will be used (request_id is returned in initial POST body)
-      - REDIS_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:results
-      ## You can force a channel. If not set, megalinter:pubsub:linter_results:REQUEST_ID will be used (request_id is returned in initial POST body)
-      - REDIS_LINTER_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:linter_results  
+      # PubSub mode (by default so commented)
+      # - REDIS_REPORTER_METHOD=PUBSUB
+      # - REDIS_LINTER_REPORTER_METHOD=PUBSUB
+      # - REDIS_ERRORS_REPORTER_METHOD=PUBSUB
+      ## You can force a channel. If not set, megalinter:pubsub:REQUEST_ID will be used (request_id is returned in initial POST body)
+      # - REDIS_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:results
+      # - REDIS_LINTER_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:linter_results 
+      # - REDIS_ERRORS_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:errors 
     depends_on:
       - megalinter_server_redis
     links:
       - megalinter_server_redis
     volumes:
-      - ./server-files:/tmp/server-files
+      - ./server-files:/tmp/server-files

server/docker-compose.yml

@@ -52,13 +52,14 @@ services:
       #- REDIS_LINTER_REPORTER_METHOD=STREAM
       #- REDIS_LINTER_REPORTER_STREAM=megalinter:stream:linter_results
 
-      # PubSub mode
-      - REDIS_REPORTER_METHOD=PUBSUB
-      - REDIS_LINTER_REPORTER_METHOD=PUBSUB
-      ## You can force a channel. If not set, megalinter:pubsub:results:REQUEST_ID will be used (request_id is returned in initial POST body)
-      - REDIS_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:results
-      ## You can force a channel. If not set, megalinter:pubsub:linter_results:REQUEST_ID will be used (request_id is returned in initial POST body)
-      - REDIS_LINTER_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:linter_results  
+      # PubSub mode (by default so commented)
+      # - REDIS_REPORTER_METHOD=PUBSUB
+      # - REDIS_LINTER_REPORTER_METHOD=PUBSUB
+      # - REDIS_ERRORS_REPORTER_METHOD=PUBSUB
+      ## You can force a channel. If not set, megalinter:pubsub:REQUEST_ID will be used (request_id is returned in initial POST body)
+      # - REDIS_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:results
+      # - REDIS_LINTER_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:linter_results 
+      # - REDIS_ERRORS_REPORTER_PUBSUB_CHANNEL=megalinter:pubsub:error
     depends_on:
       - megalinter_server_redis
     links:

server/errors.py

@@ -0,0 +1,76 @@
+import json
+import logging
+from redis import Redis
+
+from megalinter import config
+from megalinter.utils_reporter import manage_redis_stream
+
+
+class MegalinterServerException(Exception):
+    redis_host: str | None = None
+    redis_port: int | None = None
+    redis_method: str | None = None  # Stream or PubSub
+    stream_key: str | None = None
+    pubsub_channel: str | None = None
+    message_data: object | None = None
+
+    def __init__(self, message, error_code, request_id, error_details={}):
+        # Call the base class constructor with the parameters it needs
+        super().__init__(message)
+        # Now for your custom code...
+        self.error_code = error_code
+        self.message = message
+        self.request_id = request_id
+        self.error_details = error_details
+        if config.get(None, "REDIS_REPORTER", "false") == "true" and config.exists(
+            None, "REDIS_REPORTER_HOST"
+        ):
+            self.redis_host = config.get(None, "REDIS_REPORTER_HOST")
+            self.redis_port = int(config.get(None, "REDIS_REPORTER_PORT", 6379))
+            self.redis_method = config.get(
+                None, "REDIS_ERRORS_REPORTER_METHOD", "PUBSUB"
+            )
+            # Use Redis Stream
+            if self.redis_method == "STREAM":
+                self.stream_key = config.get(
+                    None,
+                    "REDIS_ERRORS_REPORTER_STREAM",
+                    "megalinter:stream:errors",
+                )
+            else:
+                # Use redis PubSub
+                self.pubsub_channel = config.get(
+                    None,
+                    "REDIS_ERRORS_REPORTER_PUBSUB_CHANNEL",
+                    "megalinter:pubsub:" + request_id,
+                )
+
+    # Send redis message before raising exception
+    def send_redis_message(self):
+        self.message_data = {
+            "messageType": "serverError",
+            "message": self.message,
+            "errorCode": self.error_code,
+            "errorDetails": self.error_details,
+            "requestId": self.request_id,
+        }
+        final_message = manage_redis_stream(self.message_data, (self.redis_method == 'STREAM'))
+        try:
+            redis = Redis(host=self.redis_host, port=self.redis_port, db=0)
+            logging.debug("REDIS Connection: " + str(redis.info()))
+            if self.redis_method == "STREAM":
+                resp = redis.xadd(self.stream_key, final_message)
+            else:
+                resp = redis.publish(self.pubsub_channel, json.dumps(final_message))
+            logging.info("REDIS RESP" + str(resp))
+        except ConnectionError as e:
+            logging.warning(
+                f"[Redis Server Reporter] Error posting message for MegaLinter: Connection error {str(e)}"
+            )
+        except Exception as e:
+            logging.warning(
+                f"[Redis Server Reporter] Error posting message for MegaLinter: Error {str(e)}"
+            )
+            logging.warning(
+                "[Redis Server Reporter] Redis Message data: " + str(self.message_data)
+            )

server/server_worker.py

@@ -8,7 +8,8 @@
 
 import git
 from megalinter import MegaLinter
-from server.types import AnalysisRequestInput, AnalysisStatus, MegalinterServerException
+from server.errors import MegalinterServerException
+from server.types import AnalysisRequestInput, AnalysisStatus
 from pygments import lexers
 
 logger = logging.getLogger(__name__)
@@ -64,19 +65,29 @@ def initialize_files(self):
             self.init_from_snippet()
             return
         # Nothing to create a request !
-        raise MegalinterServerException(
-            "Unable to initialize files for analysis", "missingAnalysisType"
+        err = MegalinterServerException(
+            "Unable to initialize files for analysis",
+            "missingAnalysisType",
+            self.id,
+            {"request_input": self.request_input},
         )
+        err.send_redis_message()
+        raise err
 
     # Init by cloning a remote repository
     def init_from_repository(self):
         temp_dir = self.create_temp_dir()
         try:
             git.Repo.clone_from(self.request_input.repositoryUrl, temp_dir)
         except Exception as e:
-            raise MegalinterServerException(
-                f"Unable to clone repository\n{str(e)}", "gitCloneError"
+            err = MegalinterServerException(
+                f"Unable to clone repository\n{str(e)}",
+                "gitCloneError",
+                self.id,
+                {"error": str(e)},
             )
+            err.send_redis_message()
+            raise err
         print(f"Cloned {self.request_input.repositoryUrl} in temp dir {temp_dir}")
         self.workspace = temp_dir
         self.repository = self.request_input.repositoryUrl
@@ -93,21 +104,33 @@ def init_from_file_upload(self, file_upload_id):
                     zip_ref.extractall(temp_dir)
             else:
                 # No zip file
-                shutil.copytree(upload_dir, temp_dir, dirs_exist_ok = True)
+                shutil.copytree(upload_dir, temp_dir, dirs_exist_ok=True)
             print(f"Copied uploaded files from {self.id} in temp dir {temp_dir}")
             self.workspace = temp_dir
             self.repository = self.request_input.repositoryUrl
         else:
-            raise MegalinterServerException("Unable to load uploaded files for analysis", "uploadedFileNotFound")
+            err = MegalinterServerException(
+                "Unable to load uploaded files for analysis",
+                "uploadedFileNotFound",
+                self.id,
+                {"file_upload_id": file_upload_id},
+            )
+            err.send_redis_message()
+            raise err
 
     # Init from user snippet
     def init_from_snippet(self):
         # Guess language using pygments
         code_lexer = lexers.guess_lexer(self.request_input.snippet)
         if not code_lexer:
-            raise MegalinterServerException(
-                "Unable to detect language from snippet", "snippetGuessError"
+            err = MegalinterServerException(
+                "Unable to detect language from snippet",
+                "snippetGuessError",
+                self.id,
+                {"snippet": self.request_input.snippet},
             )
+            err.send_redis_message()
+            raise err
         self.snippet_language = code_lexer.name
         print(f"Guessed snipped language: {self.snippet_language}")
         # Build file name
@@ -117,9 +140,14 @@ def init_from_snippet(self):
             else:
                 snippet_file_name = code_lexer.filenames[0]
         else:
-            raise MegalinterServerException(
-                f"Unable build file from {code_lexer.name} snippet", "snippetBuildError"
+            err = MegalinterServerException(
+                f"Unable build file from {code_lexer.name} snippet",
+                "snippetBuildError",
+                self.id,
+                {"snippet": self.request_input.snippet},
             )
+            err.send_redis_message()
+            raise err
         print(f"Snippet file name: {snippet_file_name}")
         temp_dir = self.create_temp_dir()
         snippet_file = os.path.join(temp_dir, snippet_file_name)

server/types.py

@@ -65,9 +65,4 @@ class AnalysisStatus(StrEnum):
     COMPLETE = "complete"
 
 
-class MegalinterServerException(Exception):
-    def __init__(self, message, error_code):
-        # Call the base class constructor with the parameters it needs
-        super().__init__(message)
-        # Now for your custom code...
-        self.code = error_code
+