Fix: set logger to stdout

aztk_cli/logger.py

@@ -1,10 +1,11 @@
 import logging
 import platform
+import sys
 
 root = logging.getLogger("aztk")
 
 DEFAULT_FORMAT = '%(message)s'
-VERBOSE_FORMAT = '%(asctime)s: %(levelname)s: %(message)s'
+VERBOSE_FORMAT = '[%(asctime)s] [%(filename)s:%(module)s:%(funcName)s:%(lineno)d] %(levelname)s - %(message)s'
 
 
 def add_coloring_to_emit_windows(fn):
@@ -61,6 +62,8 @@ def new(*args):
             color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY
         elif levelno >= 20:
             color = FOREGROUND_GREEN
+        elif levelno >= 19:
+            color = FOREGROUND_WHITE
         elif levelno >= 10:
             color = FOREGROUND_MAGENTA
         else:
@@ -86,6 +89,8 @@ def new(*args):
             color = '\x1b[33m'  # yellow
         elif levelno >= 20:
             color = '\x1b[32m'  # green
+        elif levelno >= 19:
+            color = '\x1b[0m'   # normal
         elif levelno >= 10:
             color = '\x1b[35m'  # pink
         else:
@@ -104,10 +109,20 @@ def new(*args):
     logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
 
 
+logging.PRINT = 19
+logging.addLevelName(logging.PRINT, "PRINT")
+
+def print_level(self, message, *args, **kwargs):
+    self._log(logging.PRINT, message, args, **kwargs)
+
+
 def setup_logging(verbose=False):
     if verbose:
         root.setLevel(logging.DEBUG)
-        logging.basicConfig(format=VERBOSE_FORMAT, datefmt="%Y-%m-%d  %H:%M:%S")
+        logging.basicConfig(format=VERBOSE_FORMAT, datefmt="%Y-%m-%d  %H:%M:%S", stream=sys.stdout)
     else:
-        root.setLevel(logging.INFO)
-        logging.basicConfig(format=DEFAULT_FORMAT)
+        root.setLevel(logging.PRINT)
+        logging.basicConfig(format=DEFAULT_FORMAT, stream=sys.stdout)
+
+    # add custom levels
+    logging.Logger.print = print_level

aztk_cli/spark/endpoints/cluster/cluster_app_logs.py

@@ -3,7 +3,7 @@
 import typing
 
 import aztk
-from aztk_cli import config, utils
+from aztk_cli import config, utils, log
 
 
 def setup_parser(parser: argparse.ArgumentParser):
@@ -36,4 +36,4 @@ def execute(args: typing.NamedTuple):
                 with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f:
                     f.write(app_log.log)
         else:
-            print(app_log.log)
+            log.print(app_log.log)

aztk_cli/spark/endpoints/cluster/cluster_copy.py

@@ -3,7 +3,7 @@
 import typing
 
 import aztk.spark
-from aztk_cli import config, utils
+from aztk_cli import config, log, utils
 
 
 def setup_parser(parser: argparse.ArgumentParser):
@@ -35,10 +35,10 @@ def execute(args: typing.NamedTuple):
 
 
 def print_copy_result(node_id, success, err):
-    print("-" * (len(node_id) + 6))
-    print("| ", node_id, " |")
-    print("-" * (len(node_id) + 6))
+    log.print("-" * (len(node_id) + 6))
+    log.print("| %s |", node_id)
+    log.print("-" * (len(node_id) + 6))
     if success:
-        print("Copy successful")
+        log.print("Copy successful")
     else:
-        print(err)
+        log.print(err)

aztk_cli/spark/endpoints/cluster/cluster_run.py

@@ -2,7 +2,7 @@
 import typing
 
 import aztk.spark
-from aztk_cli import config, utils
+from aztk_cli import config, log, utils
 
 
 def setup_parser(parser: argparse.ArgumentParser):

aztk_cli/spark/endpoints/job/get_app_logs.py

@@ -3,7 +3,7 @@
 import typing
 
 import aztk.spark
-from aztk_cli import config, utils
+from aztk_cli import config, log, utils
 
 
 def setup_parser(parser: argparse.ArgumentParser):
@@ -28,4 +28,4 @@ def execute(args: typing.NamedTuple):
             with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f:
                 f.write(app_log.log)
     else:
-        print(app_log.log)
+        log.print(app_log.log)

aztk_cli/spark/endpoints/job/stop.py

@@ -3,7 +3,7 @@
 import typing
 
 import aztk.spark
-from aztk_cli import config, utils
+from aztk_cli import config, log, utils
 
 
 def setup_parser(parser: argparse.ArgumentParser):
@@ -16,4 +16,4 @@ def setup_parser(parser: argparse.ArgumentParser):
 def execute(args: typing.NamedTuple):
     spark_client = aztk.spark.Client(config.load_aztk_secrets())
     spark_client.stop_job(args.job_id)
-    print("Stopped Job {0}".format(args.job_id))
+    log.print("Stopped Job {0}".format(args.job_id))

aztk_cli/utils.py

@@ -4,7 +4,6 @@
 import sys
 import threading
 import time
-import yaml
 from subprocess import call
 from typing import List
 
@@ -131,7 +130,7 @@ def stream_logs(client, cluster_id, application_name):
             application_name=application_name,
             tail=True,
             current_bytes=current_bytes)
-        print(app_logs.log, end="")
+        log.print(app_logs.log)
         if app_logs.application_state == 'completed':
             return app_logs.exit_code
         current_bytes = app_logs.total_bytes
@@ -466,4 +465,4 @@ def log_execute_result(node_id, result):
         log.info("%s\n", result)
     else:
         for line in result:
-            print(line)
+            log.print(line)