Skip to content

Commit

Permalink
Remove unneeded DateTimeSerialization class. Remove separate node_info
Browse files Browse the repository at this point in the history
in the event dictionary and leave it in the event data dictionary.
Bump log version to 2.
  • Loading branch information
gshank committed Jan 27, 2022
1 parent 53fccb3 commit 9b3a17b
Show file tree
Hide file tree
Showing 4 changed files with 7 additions and 27 deletions.
8 changes: 1 addition & 7 deletions core/dbt/events/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from collections import deque

global LOG_VERSION
LOG_VERSION = 1
LOG_VERSION = 2

# create the global event history buffer with the default max size (10k)
# python 3.7 doesn't support type hints on globals, but mypy requires them. hence the ignore.
Expand Down Expand Up @@ -157,7 +157,6 @@ def event_to_serializable_dict(
e: T_Event,
) -> Dict[str, Any]:

node_info: Dict[str, Any] = dict()
log_line = dict()
code: str
try:
Expand All @@ -172,10 +171,6 @@ def event_to_serializable_dict(
if 'code' in log_line:
del log_line['code']

# We could just leave the 'node_info' in the data field instead
if 'node_info' in log_line and log_line['node_info']:
node_info = log_line['node_info'].pop()

event_dict = {
'type': 'log_line',
'log_version': LOG_VERSION,
Expand All @@ -186,7 +181,6 @@ def event_to_serializable_dict(
'data': log_line,
'invocation_id': e.get_invocation_id(),
'thread_name': e.get_thread_name(),
'node_info': node_info,
'code': e.code
}

Expand Down
22 changes: 4 additions & 18 deletions core/dbt/events/serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
BaseConfig as MashBaseConfig
)
from mashumaro.types import SerializationStrategy
from datetime import datetime
from dateutil.parser import parse
from typing import cast


# The dbtClassMixin serialization class has a DateTime serialization strategy
# class. If a datetime ends up in an event class, we could use a similar class
# here to serialize it in our preferred format.

class ExceptionSerialization(SerializationStrategy):
def serialize(self, value):
out = str(value)
Expand All @@ -17,20 +18,6 @@ def deserialize(self, value):
return (Exception(value))


class DateTimeSerialization(SerializationStrategy):
def serialize(self, value):
out = value.isoformat()
# Assume UTC if timezone is missing
if value.tzinfo is None:
out += "Z"
return out

def deserialize(self, value):
return (
value if isinstance(value, datetime) else parse(cast(str, value))
)


class BaseExceptionSerialization(SerializationStrategy):
def serialize(self, value):
return str(value)
Expand All @@ -47,6 +34,5 @@ class EventSerialization(DataClassDictMixin):
class Config(MashBaseConfig):
serialization_strategy = {
Exception: ExceptionSerialization(),
datetime: DateTimeSerialization(),
BaseException: ExceptionSerialization(),
}
1 change: 1 addition & 0 deletions core/dbt/events/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2050,6 +2050,7 @@ def message(self) -> str:
@dataclass
class NodeFinished(DebugLevel, NodeInfo):
unique_id: str
# The following isn't a RunResult class because we run into circular imports
run_result: Dict[str, Any]
code: str = "Q024"

Expand Down
3 changes: 1 addition & 2 deletions test/interop/log_parsing/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ struct LogLine {
invocation_id: String,
thread_name: String,
data: serde_json::Value, // TODO be more specific
node_info: serde_json::Value, // TODO be more specific
}

impl ValueTest for LogLine {
Expand Down Expand Up @@ -239,7 +238,7 @@ fn test_deserialize_serialize_is_unchanged(lines: &[String]) {
mod tests {
use crate::*;

const LOG_LINE: &str = r#"{"code": "Z023", "data": {"stats": {"error": 0, "pass": 3, "skip": 0, "total": 3, "warn": 0}}, "invocation_id": "f1e1557c-4f9d-4053-bb50-572cbbf2ca64", "level": "info", "log_version": 1, "msg": "Done. PASS=3 WARN=0 ERROR=0 SKIP=0 TOTAL=3", "node_info": {}, "pid": 75854, "thread_name": "MainThread", "ts": "2021-12-03T01:32:38.334601Z", "type": "log_line"}"#;
const LOG_LINE: &str = r#"{"code": "Z023", "data": {"stats": {"error": 0, "pass": 3, "skip": 0, "total": 3, "warn": 0}}, "invocation_id": "f1e1557c-4f9d-4053-bb50-572cbbf2ca64", "level": "info", "log_version": 1, "msg": "Done. PASS=3 WARN=0 ERROR=0 SKIP=0 TOTAL=3", "pid": 75854, "thread_name": "MainThread", "ts": "2021-12-03T01:32:38.334601Z", "type": "log_line"}"#;

#[test]
fn test_basic_loop() {
Expand Down

0 comments on commit 9b3a17b

Please sign in to comment.