Skip to content

Commit

Permalink
Allow adapters to include python package logging in dbt logs (#8643)
Browse files Browse the repository at this point in the history
* add set_package_log_level functionality

* set package handler

* set package handler

* add logging about stting up logging

* test event log handler

* add event log handler

* add event log level

* rename package and add unit tests

* revert logfile config change

* cleanup and add code comments

* add changie

* swap function for dict

* add additional unit tests

* fix unit test
  • Loading branch information
colin-rogers-dbt authored and QMalcolm committed Oct 6, 2023
1 parent 0d82a31 commit 8d4100a
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 2 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20230915-123733.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: 'Allow adapters to include package logs in dbt standard logging '
time: 2023-09-15T12:37:33.862862-07:00
custom:
Author: colin-rogers-dbt
Issue: "7859"
10 changes: 9 additions & 1 deletion core/dbt/events/adapter_endpoint.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import traceback
from dataclasses import dataclass
from dbt.events.functions import fire_event
from dbt.events.functions import fire_event, EVENT_MANAGER
from dbt.events.contextvars import get_node_info
from dbt.events.event_handler import set_package_logging
from dbt.events.types import (
AdapterEventDebug,
AdapterEventInfo,
Expand Down Expand Up @@ -56,3 +57,10 @@ def critical(self, msg, *args) -> None:
name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
)
fire_event(event)

@staticmethod
def set_adapter_dependency_log_level(package_name, level):
"""By default, dbt suppresses non-dbt package logs. This method allows
you to set the log level for a specific package.
"""
set_package_logging(package_name, level, EVENT_MANAGER)
40 changes: 40 additions & 0 deletions core/dbt/events/event_handler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import logging
from typing import Union

from dbt.events.base_types import EventLevel
from dbt.events.types import Note

from dbt.events.eventmgr import IEventManager

_log_level_to_event_level_map = {
logging.DEBUG: EventLevel.DEBUG,
logging.INFO: EventLevel.INFO,
logging.WARN: EventLevel.WARN,
logging.WARNING: EventLevel.WARN,
logging.ERROR: EventLevel.ERROR,
logging.CRITICAL: EventLevel.ERROR,
}


class DbtEventLoggingHandler(logging.Handler):
"""A logging handler that wraps the EventManager
This allows non-dbt packages to log to the dbt event stream.
All logs are generated as "Note" events.
"""

def __init__(self, event_manager: IEventManager, level):
super().__init__(level)
self.event_manager = event_manager

def emit(self, record: logging.LogRecord):
note = Note(msg=record.getMessage())
level = _log_level_to_event_level_map[record.levelno]
self.event_manager.fire_event(e=note, level=level)


def set_package_logging(package_name: str, log_level: Union[str, int], event_mgr: IEventManager):
"""Attach dbt's custom logging handler to the package's logger."""
log = logging.getLogger(package_name)
log.setLevel(log_level)
event_handler = DbtEventLoggingHandler(event_manager=event_mgr, level=log_level)
log.addHandler(event_handler)
2 changes: 1 addition & 1 deletion core/dbt/events/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ def _get_stdout_config(
level: EventLevel,
log_cache_events: bool,
) -> LoggerConfig:

return LoggerConfig(
name="stdout_log",
level=level,
Expand Down Expand Up @@ -127,6 +126,7 @@ def _get_logfile_config(
level: EventLevel,
log_file_max_bytes: int,
) -> LoggerConfig:

return LoggerConfig(
name="file_log",
line_format=line_format,
Expand Down
40 changes: 40 additions & 0 deletions tests/unit/test_event_handler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import logging

from dbt.events.base_types import EventLevel
from dbt.events.event_handler import DbtEventLoggingHandler, set_package_logging
from dbt.events.eventmgr import TestEventManager


def test_event_logging_handler_emits_records_correctly():
event_manager = TestEventManager()
handler = DbtEventLoggingHandler(event_manager=event_manager, level=logging.DEBUG)
log = logging.getLogger("test")
log.setLevel(logging.DEBUG)
log.addHandler(handler)

log.debug("test")
log.info("test")
log.warning("test")
log.error("test")
log.exception("test")
log.critical("test")
assert len(event_manager.event_history) == 6
assert event_manager.event_history[0][1] == EventLevel.DEBUG
assert event_manager.event_history[1][1] == EventLevel.INFO
assert event_manager.event_history[2][1] == EventLevel.WARN
assert event_manager.event_history[3][1] == EventLevel.ERROR
assert event_manager.event_history[4][1] == EventLevel.ERROR
assert event_manager.event_history[5][1] == EventLevel.ERROR


def test_set_package_logging_sets_level_correctly():
event_manager = TestEventManager()
log = logging.getLogger("test")
set_package_logging("test", logging.DEBUG, event_manager)
log.debug("debug")
assert len(event_manager.event_history) == 1
set_package_logging("test", logging.WARN, event_manager)
log.debug("debug 2")
assert len(event_manager.event_history) == 1
log.warning("warning")
assert len(event_manager.event_history) == 3 # warning logs create two events
7 changes: 7 additions & 0 deletions tests/unit/test_events.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import logging
import re
from argparse import Namespace
from typing import TypeVar
Expand Down Expand Up @@ -83,6 +84,12 @@ def test_formatting(self):
event = types.JinjaLogDebug(msg=[1, 2, 3])
assert isinstance(event.msg, str)

def test_set_adapter_dependency_log_level(self):
logger = AdapterLogger("dbt_tests")
package_log = logging.getLogger("test_package_log")
logger.set_adapter_dependency_log_level("test_package_log", "DEBUG")
package_log.debug("debug message")


class TestEventCodes:

Expand Down

0 comments on commit 8d4100a

Please sign in to comment.