314 lines
10 KiB
Python
314 lines
10 KiB
Python
|
from abc import abstractmethod
|
||
|
|
from copy import deepcopy
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from typing import List, Iterator, Dict, Any, TypeVar, Generic
|
||
|
|
|
||
|
|
from dbt.config import RuntimeConfig, Project, IsFQNResource
|
||
|
|
from dbt.contracts.graph.model_config import BaseConfig, get_config_for
|
||
|
|
from dbt.exceptions import InternalException
|
||
|
|
from dbt.node_types import NodeType
|
||
|
|
from dbt.utils import fqn_search
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class ModelParts(IsFQNResource):
|
||
|
|
fqn: List[str]
|
||
|
|
resource_type: NodeType
|
||
|
|
package_name: str
|
||
|
|
|
||
|
|
|
||
|
|
T = TypeVar('T') # any old type
|
||
|
|
C = TypeVar('C', bound=BaseConfig)
|
||
|
|
|
||
|
|
|
||
|
|
class ConfigSource:
|
||
|
|
def __init__(self, project):
|
||
|
|
self.project = project
|
||
|
|
|
||
|
|
def get_config_dict(self, resource_type: NodeType):
|
||
|
|
...
|
||
|
|
|
||
|
|
|
||
|
|
class UnrenderedConfig(ConfigSource):
|
||
|
|
def __init__(self, project: Project):
|
||
|
|
self.project = project
|
||
|
|
|
||
|
|
def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]:
|
||
|
|
unrendered = self.project.unrendered.project_dict
|
||
|
|
if resource_type == NodeType.Seed:
|
||
|
|
model_configs = unrendered.get('seeds')
|
||
|
|
elif resource_type == NodeType.Snapshot:
|
||
|
|
model_configs = unrendered.get('snapshots')
|
||
|
|
elif resource_type == NodeType.Source:
|
||
|
|
model_configs = unrendered.get('sources')
|
||
|
|
elif resource_type == NodeType.Test:
|
||
|
|
model_configs = unrendered.get('tests')
|
||
|
|
else:
|
||
|
|
model_configs = unrendered.get('models')
|
||
|
|
|
||
|
|
if model_configs is None:
|
||
|
|
return {}
|
||
|
|
else:
|
||
|
|
return model_configs
|
||
|
|
|
||
|
|
|
||
|
|
class RenderedConfig(ConfigSource):
|
||
|
|
def __init__(self, project: Project):
|
||
|
|
self.project = project
|
||
|
|
|
||
|
|
def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]:
|
||
|
|
if resource_type == NodeType.Seed:
|
||
|
|
model_configs = self.project.seeds
|
||
|
|
elif resource_type == NodeType.Snapshot:
|
||
|
|
model_configs = self.project.snapshots
|
||
|
|
elif resource_type == NodeType.Source:
|
||
|
|
model_configs = self.project.sources
|
||
|
|
elif resource_type == NodeType.Test:
|
||
|
|
model_configs = self.project.tests
|
||
|
|
else:
|
||
|
|
model_configs = self.project.models
|
||
|
|
return model_configs
|
||
|
|
|
||
|
|
|
||
|
|
class BaseContextConfigGenerator(Generic[T]):
|
||
|
|
def __init__(self, active_project: RuntimeConfig):
|
||
|
|
self._active_project = active_project
|
||
|
|
|
||
|
|
def get_config_source(self, project: Project) -> ConfigSource:
|
||
|
|
return RenderedConfig(project)
|
||
|
|
|
||
|
|
def get_node_project(self, project_name: str):
|
||
|
|
if project_name == self._active_project.project_name:
|
||
|
|
return self._active_project
|
||
|
|
dependencies = self._active_project.load_dependencies()
|
||
|
|
if project_name not in dependencies:
|
||
|
|
raise InternalException(
|
||
|
|
f'Project name {project_name} not found in dependencies '
|
||
|
|
f'(found {list(dependencies)})'
|
||
|
|
)
|
||
|
|
return dependencies[project_name]
|
||
|
|
|
||
|
|
def _project_configs(
|
||
|
|
self, project: Project, fqn: List[str], resource_type: NodeType
|
||
|
|
) -> Iterator[Dict[str, Any]]:
|
||
|
|
src = self.get_config_source(project)
|
||
|
|
model_configs = src.get_config_dict(resource_type)
|
||
|
|
for level_config in fqn_search(model_configs, fqn):
|
||
|
|
result = {}
|
||
|
|
for key, value in level_config.items():
|
||
|
|
if key.startswith('+'):
|
||
|
|
result[key[1:].strip()] = deepcopy(value)
|
||
|
|
elif not isinstance(value, dict):
|
||
|
|
result[key] = deepcopy(value)
|
||
|
|
|
||
|
|
yield result
|
||
|
|
|
||
|
|
def _active_project_configs(
|
||
|
|
self, fqn: List[str], resource_type: NodeType
|
||
|
|
) -> Iterator[Dict[str, Any]]:
|
||
|
|
return self._project_configs(self._active_project, fqn, resource_type)
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def _update_from_config(
|
||
|
|
self, result: T, partial: Dict[str, Any], validate: bool = False
|
||
|
|
) -> T:
|
||
|
|
...
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def initial_result(self, resource_type: NodeType, base: bool) -> T:
|
||
|
|
...
|
||
|
|
|
||
|
|
def calculate_node_config(
|
||
|
|
self,
|
||
|
|
config_call_dict: Dict[str, Any],
|
||
|
|
fqn: List[str],
|
||
|
|
resource_type: NodeType,
|
||
|
|
project_name: str,
|
||
|
|
base: bool,
|
||
|
|
patch_config_dict: Dict[str, Any] = None
|
||
|
|
) -> BaseConfig:
|
||
|
|
own_config = self.get_node_project(project_name)
|
||
|
|
|
||
|
|
result = self.initial_result(resource_type=resource_type, base=base)
|
||
|
|
|
||
|
|
project_configs = self._project_configs(own_config, fqn, resource_type)
|
||
|
|
for fqn_config in project_configs:
|
||
|
|
result = self._update_from_config(result, fqn_config)
|
||
|
|
|
||
|
|
# When schema files patch config, it has lower precedence than
|
||
|
|
# config in the models (config_call_dict), so we add the patch_config_dict
|
||
|
|
# before the config_call_dict
|
||
|
|
if patch_config_dict:
|
||
|
|
result = self._update_from_config(result, patch_config_dict)
|
||
|
|
|
||
|
|
# config_calls are created in the 'experimental' model parser and
|
||
|
|
# the ParseConfigObject (via add_config_call)
|
||
|
|
result = self._update_from_config(result, config_call_dict)
|
||
|
|
|
||
|
|
if own_config.project_name != self._active_project.project_name:
|
||
|
|
for fqn_config in self._active_project_configs(fqn, resource_type):
|
||
|
|
result = self._update_from_config(result, fqn_config)
|
||
|
|
|
||
|
|
# this is mostly impactful in the snapshot config case
|
||
|
|
return result
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def calculate_node_config_dict(
|
||
|
|
self,
|
||
|
|
config_call_dict: Dict[str, Any],
|
||
|
|
fqn: List[str],
|
||
|
|
resource_type: NodeType,
|
||
|
|
project_name: str,
|
||
|
|
base: bool,
|
||
|
|
patch_config_dict: Dict[str, Any],
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
...
|
||
|
|
|
||
|
|
|
||
|
|
class ContextConfigGenerator(BaseContextConfigGenerator[C]):
|
||
|
|
def __init__(self, active_project: RuntimeConfig):
|
||
|
|
self._active_project = active_project
|
||
|
|
|
||
|
|
def get_config_source(self, project: Project) -> ConfigSource:
|
||
|
|
return RenderedConfig(project)
|
||
|
|
|
||
|
|
def initial_result(self, resource_type: NodeType, base: bool) -> C:
|
||
|
|
# defaults, own_config, config calls, active_config (if != own_config)
|
||
|
|
config_cls = get_config_for(resource_type, base=base)
|
||
|
|
# Calculate the defaults. We don't want to validate the defaults,
|
||
|
|
# because it might be invalid in the case of required config members
|
||
|
|
# (such as on snapshots!)
|
||
|
|
result = config_cls.from_dict({})
|
||
|
|
return result
|
||
|
|
|
||
|
|
def _update_from_config(
|
||
|
|
self, result: C, partial: Dict[str, Any], validate: bool = False
|
||
|
|
) -> C:
|
||
|
|
translated = self._active_project.credentials.translate_aliases(
|
||
|
|
partial
|
||
|
|
)
|
||
|
|
return result.update_from(
|
||
|
|
translated,
|
||
|
|
self._active_project.credentials.type,
|
||
|
|
validate=validate
|
||
|
|
)
|
||
|
|
|
||
|
|
def calculate_node_config_dict(
|
||
|
|
self,
|
||
|
|
config_call_dict: Dict[str, Any],
|
||
|
|
fqn: List[str],
|
||
|
|
resource_type: NodeType,
|
||
|
|
project_name: str,
|
||
|
|
base: bool,
|
||
|
|
patch_config_dict: dict = None
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
config = self.calculate_node_config(
|
||
|
|
config_call_dict=config_call_dict,
|
||
|
|
fqn=fqn,
|
||
|
|
resource_type=resource_type,
|
||
|
|
project_name=project_name,
|
||
|
|
base=base,
|
||
|
|
patch_config_dict=patch_config_dict
|
||
|
|
)
|
||
|
|
finalized = config.finalize_and_validate()
|
||
|
|
return finalized.to_dict(omit_none=True)
|
||
|
|
|
||
|
|
|
||
|
|
class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]):
|
||
|
|
def get_config_source(self, project: Project) -> ConfigSource:
|
||
|
|
return UnrenderedConfig(project)
|
||
|
|
|
||
|
|
def calculate_node_config_dict(
|
||
|
|
self,
|
||
|
|
config_call_dict: Dict[str, Any],
|
||
|
|
fqn: List[str],
|
||
|
|
resource_type: NodeType,
|
||
|
|
project_name: str,
|
||
|
|
base: bool,
|
||
|
|
patch_config_dict: dict = None
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
return self.calculate_node_config(
|
||
|
|
config_call_dict=config_call_dict,
|
||
|
|
fqn=fqn,
|
||
|
|
resource_type=resource_type,
|
||
|
|
project_name=project_name,
|
||
|
|
base=base,
|
||
|
|
patch_config_dict=patch_config_dict
|
||
|
|
)
|
||
|
|
|
||
|
|
def initial_result(
|
||
|
|
self,
|
||
|
|
resource_type: NodeType,
|
||
|
|
base: bool
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
return {}
|
||
|
|
|
||
|
|
def _update_from_config(
|
||
|
|
self,
|
||
|
|
result: Dict[str, Any],
|
||
|
|
partial: Dict[str, Any],
|
||
|
|
validate: bool = False,
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
translated = self._active_project.credentials.translate_aliases(
|
||
|
|
partial
|
||
|
|
)
|
||
|
|
result.update(translated)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
class ContextConfig:
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
active_project: RuntimeConfig,
|
||
|
|
fqn: List[str],
|
||
|
|
resource_type: NodeType,
|
||
|
|
project_name: str,
|
||
|
|
) -> None:
|
||
|
|
self._config_call_dict: Dict[str, Any] = {}
|
||
|
|
self._active_project = active_project
|
||
|
|
self._fqn = fqn
|
||
|
|
self._resource_type = resource_type
|
||
|
|
self._project_name = project_name
|
||
|
|
|
||
|
|
def add_config_call(self, opts: Dict[str, Any]) -> None:
|
||
|
|
dct = self._config_call_dict
|
||
|
|
self._add_config_call(dct, opts)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _add_config_call(cls, config_call_dict, opts: Dict[str, Any]) -> None:
|
||
|
|
for k, v in opts.items():
|
||
|
|
# MergeBehavior for post-hook and pre-hook is to collect all
|
||
|
|
# values, instead of overwriting
|
||
|
|
if k in BaseConfig.mergebehavior['append']:
|
||
|
|
if not isinstance(v, list):
|
||
|
|
v = [v]
|
||
|
|
if k in BaseConfig.mergebehavior['update'] and not isinstance(v, dict):
|
||
|
|
raise InternalException(f'expected dict, got {v}')
|
||
|
|
if k in config_call_dict and isinstance(config_call_dict[k], list):
|
||
|
|
config_call_dict[k].extend(v)
|
||
|
|
elif k in config_call_dict and isinstance(config_call_dict[k], dict):
|
||
|
|
config_call_dict[k].update(v)
|
||
|
|
else:
|
||
|
|
config_call_dict[k] = v
|
||
|
|
|
||
|
|
def build_config_dict(
|
||
|
|
self,
|
||
|
|
base: bool = False,
|
||
|
|
*,
|
||
|
|
rendered: bool = True,
|
||
|
|
patch_config_dict: dict = None
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
if rendered:
|
||
|
|
src = ContextConfigGenerator(self._active_project)
|
||
|
|
else:
|
||
|
|
src = UnrenderedConfigGenerator(self._active_project)
|
||
|
|
|
||
|
|
return src.calculate_node_config_dict(
|
||
|
|
config_call_dict=self._config_call_dict,
|
||
|
|
fqn=self._fqn,
|
||
|
|
resource_type=self._resource_type,
|
||
|
|
project_name=self._project_name,
|
||
|
|
base=base,
|
||
|
|
patch_config_dict=patch_config_dict
|
||
|
|
)
|