Store when default env vars are used in manifest

.changes/unreleased/Under the Hood-20220802-112936.yaml

@@ -0,0 +1,8 @@
+kind: Under the Hood
+body: Save use of default env vars to manifest to enable partial parsing in those
+  cases.
+time: 2022-08-02T11:29:36.417589-05:00
+custom:
+  Author: emmyoop
+  Issue: "5155"
+  PR: "5589"

core/dbt/config/renderer.py

@@ -3,13 +3,13 @@
 import os
 
 from dbt.clients.jinja import get_rendered, catch_jinja
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.context.target import TargetContext
 from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
 from dbt.context.base import BaseContext
 from dbt.contracts.connection import HasCredentials
 from dbt.exceptions import DbtProjectError, CompilationException, RecursionException
 from dbt.utils import deep_map_render
-from dbt.logger import SECRET_ENV_PREFIX
 
 
 Keypath = Tuple[Union[str, int], ...]

core/dbt/constants.py

@@ -0,0 +1,2 @@
+SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
+DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"

core/dbt/context/base.py

@@ -6,6 +6,7 @@
 from dbt import tracking
 from dbt.clients.jinja import get_rendered
 from dbt.clients.yaml_helper import yaml, safe_load, SafeLoader, Loader, Dumper  # noqa: F401
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.contracts.graph.compiled import CompiledResource
 from dbt.exceptions import (
     CompilationException,
@@ -14,7 +15,6 @@
     raise_parsing_error,
     disallow_secret_env_var,
 )
-from dbt.logger import SECRET_ENV_PREFIX
 from dbt.events.functions import fire_event, get_invocation_id
 from dbt.events.types import MacroEventInfo, MacroEventDebug
 from dbt.version import __version__ as dbt_version
@@ -305,7 +305,12 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
             return_value = default
 
         if return_value is not None:
-            self.env_vars[var] = return_value
+            # If the environment variable is set from a default, store a string indicating
+            # that so we can skip partial parsing.  Otherwise the file will be scheduled for
+            # reparsing. If the default changes, the file will have been updated and therefore
+            # will be scheduled for reparsing anyways.
+            self.env_vars[var] = return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+
             return return_value
         else:
             msg = f"Env var required but not provided: '{var}'"

core/dbt/context/configured.py

@@ -1,8 +1,8 @@
 import os
 from typing import Any, Dict, Optional
 
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.contracts.connection import AdapterRequiredConfig
-from dbt.logger import SECRET_ENV_PREFIX
 from dbt.node_types import NodeType
 from dbt.utils import MultiDict
 
@@ -94,7 +94,14 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
 
         if return_value is not None:
             if self.schema_yaml_vars:
-                self.schema_yaml_vars.env_vars[var] = return_value
+                # If the environment variable is set from a default, store a string indicating
+                # that so we can skip partial parsing.  Otherwise the file will be scheduled for
+                # reparsing. If the default changes, the file will have been updated and therefore
+                # will be scheduled for reparsing anyways.
+                self.schema_yaml_vars.env_vars[var] = (
+                    return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+                )
+
             return return_value
         else:
             msg = f"Env var required but not provided: '{var}'"

core/dbt/context/providers.py

@@ -22,7 +22,7 @@
 from .base import contextmember, contextproperty, Var
 from .configured import FQNLookup
 from .context_config import ContextConfig
-from dbt.logger import SECRET_ENV_PREFIX
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
 from .macros import MacroNamespaceBuilder, MacroNamespace
 from .manifest import ManifestContext
@@ -1211,7 +1211,14 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
             # Save the env_var value in the manifest and the var name in the source_file.
             # If this is compiling, do not save because it's irrelevant to parsing.
             if self.model and not hasattr(self.model, "compiled"):
-                self.manifest.env_vars[var] = return_value
+                # If the environment variable is set from a default, store a string indicating
+                # that so we can skip partial parsing.  Otherwise the file will be scheduled for
+                # reparsing. If the default changes, the file will have been updated and therefore
+                # will be scheduled for reparsing anyways.
+                self.manifest.env_vars[var] = (
+                    return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+                )
+
                 # hooks come from dbt_project.yml which doesn't have a real file_id
                 if self.model.file_id in self.manifest.files:
                     source_file = self.manifest.files[self.model.file_id]
@@ -1535,7 +1542,13 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
         if return_value is not None:
             # Save the env_var value in the manifest and the var name in the source_file
             if self.model:
-                self.manifest.env_vars[var] = return_value
+                # If the environment variable is set from a default, store a string indicating
+                # that so we can skip partial parsing.  Otherwise the file will be scheduled for
+                # reparsing. If the default changes, the file will have been updated and therefore
+                # will be scheduled for reparsing anyways.
+                self.manifest.env_vars[var] = (
+                    return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+                )
                 # the "model" should only be test nodes, but just in case, check
                 # TODO CT-211
                 if self.model.resource_type == NodeType.Test and self.model.file_key_name:  # type: ignore[union-attr] # noqa

core/dbt/context/secret.py

@@ -3,8 +3,8 @@
 
 from .base import BaseContext, contextmember
 
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.exceptions import raise_parsing_error
-from dbt.logger import SECRET_ENV_PREFIX
 
 
 SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
@@ -43,7 +43,11 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
             # if it's a 'secret' env var, we shouldn't even get here
             # but just to be safe — don't save secrets
             if not var.startswith(SECRET_ENV_PREFIX):
-                self.env_vars[var] = return_value
+                # If the environment variable is set from a default, store a string indicating
+                # that so we can skip partial parsing.  Otherwise the file will be scheduled for
+                # reparsing. If the default changes, the file will have been updated and therefore
+                # will be scheduled for reparsing anyways.
+                self.env_vars[var] = return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
             return return_value
         else:
             msg = f"Env var required but not provided: '{var}'"

core/dbt/events/functions.py

@@ -3,9 +3,10 @@
 from dbt.events.base_types import NoStdOut, Event, NoFile, ShowException, Cache
 from dbt.events.types import EventBufferFull, T_Event, MainReportVersion, EmptyLine
 import dbt.flags as flags
+from dbt.constants import SECRET_ENV_PREFIX
 
 # TODO this will need to move eventually
-from dbt.logger import SECRET_ENV_PREFIX, make_log_dir_if_missing, GLOBAL_LOGGER
+from dbt.logger import make_log_dir_if_missing, GLOBAL_LOGGER
 from datetime import datetime
 import json
 import io

core/dbt/logger.py

@@ -13,6 +13,7 @@
 
 import colorama
 import logbook
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.dataclass_schema import dbtClassMixin
 
 # Colorama is needed for colored logs on Windows because we're using logger.info
@@ -31,8 +32,6 @@
     "{record.time:%Y-%m-%d %H:%M:%S.%f%z} " "({record.thread_name}): " "{record.message}"
 )
 
-SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
-
 
 def get_secret_env() -> List[str]:
     return [v for k, v in os.environ.items() if k.startswith(SECRET_ENV_PREFIX)]

core/dbt/parser/partial.py

@@ -21,6 +21,7 @@
     PartialParsingDeletedExposure,
     PartialParsingDeletedMetric,
 )
+from dbt.constants import DEFAULT_ENV_PLACEHOLDER
 from dbt.node_types import NodeType
 
 
@@ -961,6 +962,13 @@ def build_env_vars_to_files(self):
             prev_value = self.saved_manifest.env_vars[env_var]
             current_value = os.getenv(env_var)
             if current_value is None:
+                # This will be true when depending on the default value.
+                # We store env vars set by defaults as a static string so we can recognize they have
+                # defaults.  We depend on default changes triggering reparsing by file change. If
+                # the file has not changed we can assume the default has not changed.
+                if prev_value == DEFAULT_ENV_PLACEHOLDER:
+                    unchanged_vars.append(env_var)
+                    continue
                 # env_var no longer set, remove from manifest
                 delete_vars.append(env_var)
             if prev_value == current_value:

test/integration/068_partial_parsing_tests/test_pp_vars.py

@@ -1,8 +1,8 @@
 from dbt.exceptions import CompilationException, ParsingException
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.files import ParseFileType
 from dbt.contracts.results import TestStatus
-from dbt.logger import SECRET_ENV_PREFIX
 from dbt.parser.partial import special_override_macros
 from test.integration.base import DBTIntegrationTest, use_profile, normalize, get_manifest
 import shutil

tests/functional/context_methods/test_env_vars.py

@@ -1,8 +1,8 @@
 import pytest
 import os
 
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.tests.util import run_dbt, get_manifest, run_dbt_and_capture
-from dbt.logger import SECRET_ENV_PREFIX
 
 
 context_sql = """
@@ -36,6 +36,8 @@
     '{{ invocation_id }}'  as invocation_id,
 
     '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var,
+    '{{ env_var("DBT_TEST_IGNORE_DEFAULT", "ignored_default_val") }}' as env_var_ignore_default,
+    '{{ env_var("DBT_TEST_USE_DEFAULT", "use_my_default_val") }}' as env_var_use_default,
     'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs
     '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret
 
@@ -54,11 +56,13 @@ def setup(self):
         os.environ["DBT_TEST_PASS"] = "password"
         os.environ[SECRET_ENV_PREFIX + "SECRET"] = "secret_variable"
         os.environ["DBT_TEST_NOT_SECRET"] = "regular_variable"
+        os.environ["DBT_TEST_IGNORE_DEFAULT"] = "ignored_default"
         yield
         del os.environ["DBT_TEST_ENV_VAR"]
         del os.environ["DBT_TEST_USER"]
         del os.environ[SECRET_ENV_PREFIX + "SECRET"]
         del os.environ["DBT_TEST_NOT_SECRET"]
+        del os.environ["DBT_TEST_IGNORE_DEFAULT"]
 
     @pytest.fixture(scope="class")
     def profiles_config_update(self, unique_schema):
@@ -129,7 +133,12 @@ def test_env_vars_dev(
         ctx = self.get_ctx_vars(project)
 
         manifest = get_manifest(project.project_root)
-        expected = {"DBT_TEST_ENV_VAR": "1", "DBT_TEST_NOT_SECRET": "regular_variable"}
+        expected = {
+            "DBT_TEST_ENV_VAR": "1",
+            "DBT_TEST_NOT_SECRET": "regular_variable",
+            "DBT_TEST_IGNORE_DEFAULT": "ignored_default",
+            "DBT_TEST_USE_DEFAULT": DEFAULT_ENV_PLACEHOLDER,
+        }
         assert manifest.env_vars == expected
 
         this = '"{}"."{}"."context"'.format(project.database, project.test_schema)

tests/functional/context_methods/test_secret_env_vars.py

@@ -1,10 +1,10 @@
 import pytest
 import os
 
-from dbt.tests.util import run_dbt, run_dbt_and_capture
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.exceptions import ParsingException, InternalException
-from dbt.logger import SECRET_ENV_PREFIX
 from tests.functional.context_methods.first_dependency import FirstDependencyProject
+from dbt.tests.util import run_dbt, run_dbt_and_capture
 
 
 secret_bad__context_sql = """