1476 lines
48 KiB
Python
1476 lines
48 KiB
Python
|
import abc
|
|||
|
|
import os
|
|||
|
|
from typing import (
|
|||
|
|
Callable, Any, Dict, Optional, Union, List, TypeVar, Type, Iterable,
|
|||
|
|
Mapping,
|
|||
|
|
)
|
|||
|
|
from typing_extensions import Protocol
|
|||
|
|
|
|||
|
|
from dbt import deprecations
|
|||
|
|
from dbt.adapters.base.column import Column
|
|||
|
|
from dbt.adapters.factory import (
|
|||
|
|
get_adapter, get_adapter_package_names, get_adapter_type_names
|
|||
|
|
)
|
|||
|
|
from dbt.clients import agate_helper
|
|||
|
|
from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack
|
|||
|
|
from dbt.config import RuntimeConfig, Project
|
|||
|
|
from .base import contextmember, contextproperty, Var
|
|||
|
|
from .configured import FQNLookup
|
|||
|
|
from .context_config import ContextConfig
|
|||
|
|
from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
|
|||
|
|
from .macros import MacroNamespaceBuilder, MacroNamespace
|
|||
|
|
from .manifest import ManifestContext
|
|||
|
|
from dbt.contracts.connection import AdapterResponse
|
|||
|
|
from dbt.contracts.graph.manifest import (
|
|||
|
|
Manifest, Disabled
|
|||
|
|
)
|
|||
|
|
from dbt.contracts.graph.compiled import (
|
|||
|
|
CompiledResource,
|
|||
|
|
CompiledSeedNode,
|
|||
|
|
ManifestNode,
|
|||
|
|
)
|
|||
|
|
from dbt.contracts.graph.parsed import (
|
|||
|
|
ParsedMacro,
|
|||
|
|
ParsedExposure,
|
|||
|
|
ParsedSeedNode,
|
|||
|
|
ParsedSourceDefinition,
|
|||
|
|
)
|
|||
|
|
from dbt.exceptions import (
|
|||
|
|
CompilationException,
|
|||
|
|
InternalException,
|
|||
|
|
ValidationException,
|
|||
|
|
RuntimeException,
|
|||
|
|
missing_config,
|
|||
|
|
raise_compiler_error,
|
|||
|
|
ref_invalid_args,
|
|||
|
|
ref_target_not_found,
|
|||
|
|
ref_bad_context,
|
|||
|
|
source_target_not_found,
|
|||
|
|
wrapped_exports,
|
|||
|
|
)
|
|||
|
|
from dbt.config import IsFQNResource
|
|||
|
|
from dbt.logger import GLOBAL_LOGGER as logger # noqa
|
|||
|
|
from dbt.node_types import NodeType
|
|||
|
|
|
|||
|
|
from dbt.utils import (
|
|||
|
|
merge, AttrDict, MultiDict
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
import agate
|
|||
|
|
|
|||
|
|
|
|||
|
|
_MISSING = object()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# base classes
|
|||
|
|
class RelationProxy:
|
|||
|
|
def __init__(self, adapter):
|
|||
|
|
self._quoting_config = adapter.config.quoting
|
|||
|
|
self._relation_type = adapter.Relation
|
|||
|
|
|
|||
|
|
def __getattr__(self, key):
|
|||
|
|
return getattr(self._relation_type, key)
|
|||
|
|
|
|||
|
|
def create_from_source(self, *args, **kwargs):
|
|||
|
|
# bypass our create when creating from source so as not to mess up
|
|||
|
|
# the source quoting
|
|||
|
|
return self._relation_type.create_from_source(*args, **kwargs)
|
|||
|
|
|
|||
|
|
def create(self, *args, **kwargs):
|
|||
|
|
kwargs['quote_policy'] = merge(
|
|||
|
|
self._quoting_config,
|
|||
|
|
kwargs.pop('quote_policy', {})
|
|||
|
|
)
|
|||
|
|
return self._relation_type.create(*args, **kwargs)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class BaseDatabaseWrapper:
|
|||
|
|
"""
|
|||
|
|
Wrapper for runtime database interaction. Applies the runtime quote policy
|
|||
|
|
via a relation proxy.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, adapter, namespace: MacroNamespace):
|
|||
|
|
self._adapter = adapter
|
|||
|
|
self.Relation = RelationProxy(adapter)
|
|||
|
|
self._namespace = namespace
|
|||
|
|
|
|||
|
|
def __getattr__(self, name):
|
|||
|
|
raise NotImplementedError('subclasses need to implement this')
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def config(self):
|
|||
|
|
return self._adapter.config
|
|||
|
|
|
|||
|
|
def type(self):
|
|||
|
|
return self._adapter.type()
|
|||
|
|
|
|||
|
|
def commit(self):
|
|||
|
|
return self._adapter.commit_if_has_connection()
|
|||
|
|
|
|||
|
|
def _get_adapter_macro_prefixes(self) -> List[str]:
|
|||
|
|
# order matters for dispatch:
|
|||
|
|
# 1. current adapter
|
|||
|
|
# 2. any parent adapters (dependencies)
|
|||
|
|
# 3. 'default'
|
|||
|
|
search_prefixes = get_adapter_type_names(self._adapter.type()) + ['default']
|
|||
|
|
return search_prefixes
|
|||
|
|
|
|||
|
|
def dispatch(
|
|||
|
|
self,
|
|||
|
|
macro_name: str,
|
|||
|
|
macro_namespace: Optional[str] = None,
|
|||
|
|
packages: Optional[List[str]] = None,
|
|||
|
|
) -> MacroGenerator:
|
|||
|
|
search_packages: List[Optional[str]]
|
|||
|
|
|
|||
|
|
if '.' in macro_name:
|
|||
|
|
suggest_package, suggest_macro_name = macro_name.split('.', 1)
|
|||
|
|
msg = (
|
|||
|
|
f'In adapter.dispatch, got a macro name of "{macro_name}", '
|
|||
|
|
f'but "." is not a valid macro name component. Did you mean '
|
|||
|
|
f'`adapter.dispatch("{suggest_macro_name}", '
|
|||
|
|
f'packages=["{suggest_package}"])`?'
|
|||
|
|
)
|
|||
|
|
raise CompilationException(msg)
|
|||
|
|
|
|||
|
|
if packages is not None:
|
|||
|
|
deprecations.warn('dispatch-packages', macro_name=macro_name)
|
|||
|
|
|
|||
|
|
namespace = packages if packages else macro_namespace
|
|||
|
|
|
|||
|
|
if namespace is None:
|
|||
|
|
search_packages = [None]
|
|||
|
|
elif isinstance(namespace, str):
|
|||
|
|
search_packages = self._adapter.config.get_macro_search_order(namespace)
|
|||
|
|
if not search_packages and namespace in self._adapter.config.dependencies:
|
|||
|
|
search_packages = [self.config.project_name, namespace]
|
|||
|
|
if not search_packages:
|
|||
|
|
raise CompilationException(
|
|||
|
|
f'In adapter.dispatch, got a string packages argument '
|
|||
|
|
f'("{packages}"), but packages should be None or a list.'
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
# Not a string and not None so must be a list
|
|||
|
|
search_packages = namespace
|
|||
|
|
|
|||
|
|
attempts = []
|
|||
|
|
|
|||
|
|
for package_name in search_packages:
|
|||
|
|
for prefix in self._get_adapter_macro_prefixes():
|
|||
|
|
search_name = f'{prefix}__{macro_name}'
|
|||
|
|
try:
|
|||
|
|
# this uses the namespace from the context
|
|||
|
|
macro = self._namespace.get_from_package(
|
|||
|
|
package_name, search_name
|
|||
|
|
)
|
|||
|
|
except CompilationException:
|
|||
|
|
# Only raise CompilationException if macro is not found in
|
|||
|
|
# any package
|
|||
|
|
macro = None
|
|||
|
|
|
|||
|
|
if package_name is None:
|
|||
|
|
attempts.append(search_name)
|
|||
|
|
else:
|
|||
|
|
attempts.append(f'{package_name}.{search_name}')
|
|||
|
|
|
|||
|
|
if macro is not None:
|
|||
|
|
return macro
|
|||
|
|
|
|||
|
|
searched = ', '.join(repr(a) for a in attempts)
|
|||
|
|
msg = (
|
|||
|
|
f"In dispatch: No macro named '{macro_name}' found\n"
|
|||
|
|
f" Searched for: {searched}"
|
|||
|
|
)
|
|||
|
|
raise CompilationException(msg)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class BaseResolver(metaclass=abc.ABCMeta):
|
|||
|
|
def __init__(self, db_wrapper, model, config, manifest):
|
|||
|
|
self.db_wrapper = db_wrapper
|
|||
|
|
self.model = model
|
|||
|
|
self.config = config
|
|||
|
|
self.manifest = manifest
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def current_project(self):
|
|||
|
|
return self.config.project_name
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def Relation(self):
|
|||
|
|
return self.db_wrapper.Relation
|
|||
|
|
|
|||
|
|
@abc.abstractmethod
|
|||
|
|
def __call__(self, *args: str) -> Union[str, RelationProxy]:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
|
|||
|
|
class BaseRefResolver(BaseResolver):
|
|||
|
|
@abc.abstractmethod
|
|||
|
|
def resolve(
|
|||
|
|
self, name: str, package: Optional[str] = None
|
|||
|
|
) -> RelationProxy:
|
|||
|
|
...
|
|||
|
|
|
|||
|
|
def _repack_args(
|
|||
|
|
self, name: str, package: Optional[str]
|
|||
|
|
) -> List[str]:
|
|||
|
|
if package is None:
|
|||
|
|
return [name]
|
|||
|
|
else:
|
|||
|
|
return [package, name]
|
|||
|
|
|
|||
|
|
def validate_args(self, name: str, package: Optional[str]):
|
|||
|
|
if not isinstance(name, str):
|
|||
|
|
raise CompilationException(
|
|||
|
|
f'The name argument to ref() must be a string, got '
|
|||
|
|
f'{type(name)}'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if package is not None and not isinstance(package, str):
|
|||
|
|
raise CompilationException(
|
|||
|
|
f'The package argument to ref() must be a string or None, got '
|
|||
|
|
f'{type(package)}'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def __call__(self, *args: str) -> RelationProxy:
|
|||
|
|
name: str
|
|||
|
|
package: Optional[str] = None
|
|||
|
|
|
|||
|
|
if len(args) == 1:
|
|||
|
|
name = args[0]
|
|||
|
|
elif len(args) == 2:
|
|||
|
|
package, name = args
|
|||
|
|
else:
|
|||
|
|
ref_invalid_args(self.model, args)
|
|||
|
|
self.validate_args(name, package)
|
|||
|
|
return self.resolve(name, package)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class BaseSourceResolver(BaseResolver):
|
|||
|
|
@abc.abstractmethod
|
|||
|
|
def resolve(self, source_name: str, table_name: str):
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def validate_args(self, source_name: str, table_name: str):
|
|||
|
|
if not isinstance(source_name, str):
|
|||
|
|
raise CompilationException(
|
|||
|
|
f'The source name (first) argument to source() must be a '
|
|||
|
|
f'string, got {type(source_name)}'
|
|||
|
|
)
|
|||
|
|
if not isinstance(table_name, str):
|
|||
|
|
raise CompilationException(
|
|||
|
|
f'The table name (second) argument to source() must be a '
|
|||
|
|
f'string, got {type(table_name)}'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def __call__(self, *args: str) -> RelationProxy:
|
|||
|
|
if len(args) != 2:
|
|||
|
|
raise_compiler_error(
|
|||
|
|
f"source() takes exactly two arguments ({len(args)} given)",
|
|||
|
|
self.model
|
|||
|
|
)
|
|||
|
|
self.validate_args(args[0], args[1])
|
|||
|
|
return self.resolve(args[0], args[1])
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Config(Protocol):
|
|||
|
|
def __init__(self, model, context_config: Optional[ContextConfig]):
|
|||
|
|
...
|
|||
|
|
|
|||
|
|
|
|||
|
|
# Implementation of "config(..)" calls in models
|
|||
|
|
class ParseConfigObject(Config):
|
|||
|
|
def __init__(self, model, context_config: Optional[ContextConfig]):
|
|||
|
|
self.model = model
|
|||
|
|
self.context_config = context_config
|
|||
|
|
|
|||
|
|
def _transform_config(self, config):
|
|||
|
|
for oldkey in ('pre_hook', 'post_hook'):
|
|||
|
|
if oldkey in config:
|
|||
|
|
newkey = oldkey.replace('_', '-')
|
|||
|
|
if newkey in config:
|
|||
|
|
raise_compiler_error(
|
|||
|
|
'Invalid config, has conflicting keys "{}" and "{}"'
|
|||
|
|
.format(oldkey, newkey),
|
|||
|
|
self.model
|
|||
|
|
)
|
|||
|
|
config[newkey] = config.pop(oldkey)
|
|||
|
|
return config
|
|||
|
|
|
|||
|
|
def __call__(self, *args, **kwargs):
|
|||
|
|
if len(args) == 1 and len(kwargs) == 0:
|
|||
|
|
opts = args[0]
|
|||
|
|
elif len(args) == 0 and len(kwargs) > 0:
|
|||
|
|
opts = kwargs
|
|||
|
|
else:
|
|||
|
|
raise_compiler_error(
|
|||
|
|
"Invalid inline model config",
|
|||
|
|
self.model)
|
|||
|
|
|
|||
|
|
opts = self._transform_config(opts)
|
|||
|
|
|
|||
|
|
# it's ok to have a parse context with no context config, but you must
|
|||
|
|
# not call it!
|
|||
|
|
if self.context_config is None:
|
|||
|
|
raise RuntimeException(
|
|||
|
|
'At parse time, did not receive a context config'
|
|||
|
|
)
|
|||
|
|
self.context_config.add_config_call(opts)
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
def set(self, name, value):
|
|||
|
|
return self.__call__({name: value})
|
|||
|
|
|
|||
|
|
def require(self, name, validator=None):
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
def get(self, name, validator=None, default=None):
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
def persist_relation_docs(self) -> bool:
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def persist_column_docs(self) -> bool:
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeConfigObject(Config):
|
|||
|
|
def __init__(
|
|||
|
|
self, model, context_config: Optional[ContextConfig] = None
|
|||
|
|
):
|
|||
|
|
self.model = model
|
|||
|
|
# we never use or get a config, only the parser cares
|
|||
|
|
|
|||
|
|
def __call__(self, *args, **kwargs):
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
def set(self, name, value):
|
|||
|
|
return self.__call__({name: value})
|
|||
|
|
|
|||
|
|
def _validate(self, validator, value):
|
|||
|
|
validator(value)
|
|||
|
|
|
|||
|
|
def _lookup(self, name, default=_MISSING):
|
|||
|
|
# if this is a macro, there might be no `model.config`.
|
|||
|
|
if not hasattr(self.model, 'config'):
|
|||
|
|
result = default
|
|||
|
|
else:
|
|||
|
|
result = self.model.config.get(name, default)
|
|||
|
|
if result is _MISSING:
|
|||
|
|
missing_config(self.model, name)
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
def require(self, name, validator=None):
|
|||
|
|
to_return = self._lookup(name)
|
|||
|
|
|
|||
|
|
if validator is not None:
|
|||
|
|
self._validate(validator, to_return)
|
|||
|
|
|
|||
|
|
return to_return
|
|||
|
|
|
|||
|
|
def get(self, name, validator=None, default=None):
|
|||
|
|
to_return = self._lookup(name, default)
|
|||
|
|
|
|||
|
|
if validator is not None and default is not None:
|
|||
|
|
self._validate(validator, to_return)
|
|||
|
|
|
|||
|
|
return to_return
|
|||
|
|
|
|||
|
|
def persist_relation_docs(self) -> bool:
|
|||
|
|
persist_docs = self.get('persist_docs', default={})
|
|||
|
|
if not isinstance(persist_docs, dict):
|
|||
|
|
raise_compiler_error(
|
|||
|
|
f"Invalid value provided for 'persist_docs'. Expected dict "
|
|||
|
|
f"but received {type(persist_docs)}")
|
|||
|
|
|
|||
|
|
return persist_docs.get('relation', False)
|
|||
|
|
|
|||
|
|
def persist_column_docs(self) -> bool:
|
|||
|
|
persist_docs = self.get('persist_docs', default={})
|
|||
|
|
if not isinstance(persist_docs, dict):
|
|||
|
|
raise_compiler_error(
|
|||
|
|
f"Invalid value provided for 'persist_docs'. Expected dict "
|
|||
|
|
f"but received {type(persist_docs)}")
|
|||
|
|
|
|||
|
|
return persist_docs.get('columns', False)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# `adapter` implementations
|
|||
|
|
class ParseDatabaseWrapper(BaseDatabaseWrapper):
|
|||
|
|
"""The parser subclass of the database wrapper applies any explicit
|
|||
|
|
parse-time overrides.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __getattr__(self, name):
|
|||
|
|
override = (name in self._adapter._available_ and
|
|||
|
|
name in self._adapter._parse_replacements_)
|
|||
|
|
|
|||
|
|
if override:
|
|||
|
|
return self._adapter._parse_replacements_[name]
|
|||
|
|
elif name in self._adapter._available_:
|
|||
|
|
return getattr(self._adapter, name)
|
|||
|
|
else:
|
|||
|
|
raise AttributeError(
|
|||
|
|
"'{}' object has no attribute '{}'".format(
|
|||
|
|
self.__class__.__name__, name
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeDatabaseWrapper(BaseDatabaseWrapper):
|
|||
|
|
"""The runtime database wrapper exposes everything the adapter marks
|
|||
|
|
available.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __getattr__(self, name):
|
|||
|
|
if name in self._adapter._available_:
|
|||
|
|
return getattr(self._adapter, name)
|
|||
|
|
else:
|
|||
|
|
raise AttributeError(
|
|||
|
|
"'{}' object has no attribute '{}'".format(
|
|||
|
|
self.__class__.__name__, name
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# `ref` implementations
|
|||
|
|
class ParseRefResolver(BaseRefResolver):
|
|||
|
|
def resolve(
|
|||
|
|
self, name: str, package: Optional[str] = None
|
|||
|
|
) -> RelationProxy:
|
|||
|
|
self.model.refs.append(self._repack_args(name, package))
|
|||
|
|
|
|||
|
|
return self.Relation.create_from(self.config, self.model)
|
|||
|
|
|
|||
|
|
|
|||
|
|
ResolveRef = Union[Disabled, ManifestNode]
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeRefResolver(BaseRefResolver):
|
|||
|
|
def resolve(
|
|||
|
|
self, target_name: str, target_package: Optional[str] = None
|
|||
|
|
) -> RelationProxy:
|
|||
|
|
target_model = self.manifest.resolve_ref(
|
|||
|
|
target_name,
|
|||
|
|
target_package,
|
|||
|
|
self.current_project,
|
|||
|
|
self.model.package_name,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if target_model is None or isinstance(target_model, Disabled):
|
|||
|
|
ref_target_not_found(
|
|||
|
|
self.model,
|
|||
|
|
target_name,
|
|||
|
|
target_package,
|
|||
|
|
disabled=isinstance(target_model, Disabled),
|
|||
|
|
)
|
|||
|
|
self.validate(target_model, target_name, target_package)
|
|||
|
|
return self.create_relation(target_model, target_name)
|
|||
|
|
|
|||
|
|
def create_relation(
|
|||
|
|
self, target_model: ManifestNode, name: str
|
|||
|
|
) -> RelationProxy:
|
|||
|
|
if target_model.is_ephemeral_model:
|
|||
|
|
self.model.set_cte(target_model.unique_id, None)
|
|||
|
|
return self.Relation.create_ephemeral_from_node(
|
|||
|
|
self.config, target_model
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
return self.Relation.create_from(self.config, target_model)
|
|||
|
|
|
|||
|
|
def validate(
|
|||
|
|
self,
|
|||
|
|
resolved: ManifestNode,
|
|||
|
|
target_name: str,
|
|||
|
|
target_package: Optional[str]
|
|||
|
|
) -> None:
|
|||
|
|
if resolved.unique_id not in self.model.depends_on.nodes:
|
|||
|
|
args = self._repack_args(target_name, target_package)
|
|||
|
|
ref_bad_context(self.model, args)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class OperationRefResolver(RuntimeRefResolver):
|
|||
|
|
def validate(
|
|||
|
|
self,
|
|||
|
|
resolved: ManifestNode,
|
|||
|
|
target_name: str,
|
|||
|
|
target_package: Optional[str],
|
|||
|
|
) -> None:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def create_relation(
|
|||
|
|
self, target_model: ManifestNode, name: str
|
|||
|
|
) -> RelationProxy:
|
|||
|
|
if target_model.is_ephemeral_model:
|
|||
|
|
# In operations, we can't ref() ephemeral nodes, because
|
|||
|
|
# ParsedMacros do not support set_cte
|
|||
|
|
raise_compiler_error(
|
|||
|
|
'Operations can not ref() ephemeral nodes, but {} is ephemeral'
|
|||
|
|
.format(target_model.name),
|
|||
|
|
self.model
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
return super().create_relation(target_model, name)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# `source` implementations
|
|||
|
|
class ParseSourceResolver(BaseSourceResolver):
|
|||
|
|
def resolve(self, source_name: str, table_name: str):
|
|||
|
|
# When you call source(), this is what happens at parse time
|
|||
|
|
self.model.sources.append([source_name, table_name])
|
|||
|
|
return self.Relation.create_from(self.config, self.model)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeSourceResolver(BaseSourceResolver):
|
|||
|
|
def resolve(self, source_name: str, table_name: str):
|
|||
|
|
target_source = self.manifest.resolve_source(
|
|||
|
|
source_name,
|
|||
|
|
table_name,
|
|||
|
|
self.current_project,
|
|||
|
|
self.model.package_name,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if target_source is None or isinstance(target_source, Disabled):
|
|||
|
|
source_target_not_found(
|
|||
|
|
self.model,
|
|||
|
|
source_name,
|
|||
|
|
table_name,
|
|||
|
|
)
|
|||
|
|
return self.Relation.create_from_source(target_source)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# `var` implementations.
|
|||
|
|
class ModelConfiguredVar(Var):
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
context: Dict[str, Any],
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
node: CompiledResource,
|
|||
|
|
) -> None:
|
|||
|
|
self._node: CompiledResource
|
|||
|
|
self._config: RuntimeConfig = config
|
|||
|
|
super().__init__(context, config.cli_vars, node=node)
|
|||
|
|
|
|||
|
|
def packages_for_node(self) -> Iterable[Project]:
|
|||
|
|
dependencies = self._config.load_dependencies()
|
|||
|
|
package_name = self._node.package_name
|
|||
|
|
|
|||
|
|
if package_name != self._config.project_name:
|
|||
|
|
if package_name not in dependencies:
|
|||
|
|
# I don't think this is actually reachable
|
|||
|
|
raise_compiler_error(
|
|||
|
|
f'Node package named {package_name} not found!',
|
|||
|
|
self._node
|
|||
|
|
)
|
|||
|
|
yield dependencies[package_name]
|
|||
|
|
yield self._config
|
|||
|
|
|
|||
|
|
def _generate_merged(self) -> Mapping[str, Any]:
|
|||
|
|
search_node: IsFQNResource
|
|||
|
|
if isinstance(self._node, IsFQNResource):
|
|||
|
|
search_node = self._node
|
|||
|
|
else:
|
|||
|
|
search_node = FQNLookup(self._node.package_name)
|
|||
|
|
|
|||
|
|
adapter_type = self._config.credentials.type
|
|||
|
|
|
|||
|
|
merged = MultiDict()
|
|||
|
|
for project in self.packages_for_node():
|
|||
|
|
merged.add(project.vars.vars_for(search_node, adapter_type))
|
|||
|
|
merged.add(self._cli_vars)
|
|||
|
|
return merged
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ParseVar(ModelConfiguredVar):
|
|||
|
|
def get_missing_var(self, var_name):
|
|||
|
|
# in the parser, just always return None.
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeVar(ModelConfiguredVar):
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
|
|||
|
|
# Providers
|
|||
|
|
class Provider(Protocol):
|
|||
|
|
execute: bool
|
|||
|
|
Config: Type[Config]
|
|||
|
|
DatabaseWrapper: Type[BaseDatabaseWrapper]
|
|||
|
|
Var: Type[ModelConfiguredVar]
|
|||
|
|
ref: Type[BaseRefResolver]
|
|||
|
|
source: Type[BaseSourceResolver]
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ParseProvider(Provider):
|
|||
|
|
execute = False
|
|||
|
|
Config = ParseConfigObject
|
|||
|
|
DatabaseWrapper = ParseDatabaseWrapper
|
|||
|
|
Var = ParseVar
|
|||
|
|
ref = ParseRefResolver
|
|||
|
|
source = ParseSourceResolver
|
|||
|
|
|
|||
|
|
|
|||
|
|
class GenerateNameProvider(Provider):
|
|||
|
|
execute = False
|
|||
|
|
Config = RuntimeConfigObject
|
|||
|
|
DatabaseWrapper = ParseDatabaseWrapper
|
|||
|
|
Var = RuntimeVar
|
|||
|
|
ref = ParseRefResolver
|
|||
|
|
source = ParseSourceResolver
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeProvider(Provider):
|
|||
|
|
execute = True
|
|||
|
|
Config = RuntimeConfigObject
|
|||
|
|
DatabaseWrapper = RuntimeDatabaseWrapper
|
|||
|
|
Var = RuntimeVar
|
|||
|
|
ref = RuntimeRefResolver
|
|||
|
|
source = RuntimeSourceResolver
|
|||
|
|
|
|||
|
|
|
|||
|
|
class OperationProvider(RuntimeProvider):
|
|||
|
|
ref = OperationRefResolver
|
|||
|
|
|
|||
|
|
|
|||
|
|
T = TypeVar('T')
|
|||
|
|
|
|||
|
|
|
|||
|
|
# Base context collection, used for parsing configs.
|
|||
|
|
class ProviderContext(ManifestContext):
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
model,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
provider: Provider,
|
|||
|
|
context_config: Optional[ContextConfig],
|
|||
|
|
) -> None:
|
|||
|
|
if provider is None:
|
|||
|
|
raise InternalException(
|
|||
|
|
f"Invalid provider given to context: {provider}"
|
|||
|
|
)
|
|||
|
|
# mypy appeasement - we know it'll be a RuntimeConfig
|
|||
|
|
self.config: RuntimeConfig
|
|||
|
|
self.model: Union[ParsedMacro, ManifestNode] = model
|
|||
|
|
super().__init__(config, manifest, model.package_name)
|
|||
|
|
self.sql_results: Dict[str, AttrDict] = {}
|
|||
|
|
self.context_config: Optional[ContextConfig] = context_config
|
|||
|
|
self.provider: Provider = provider
|
|||
|
|
self.adapter = get_adapter(self.config)
|
|||
|
|
# The macro namespace is used in creating the DatabaseWrapper
|
|||
|
|
self.db_wrapper = self.provider.DatabaseWrapper(
|
|||
|
|
self.adapter, self.namespace
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# This overrides the method in ManifestContext, and provides
|
|||
|
|
# a model, which the ManifestContext builder does not
|
|||
|
|
def _get_namespace_builder(self):
|
|||
|
|
internal_packages = get_adapter_package_names(
|
|||
|
|
self.config.credentials.type
|
|||
|
|
)
|
|||
|
|
return MacroNamespaceBuilder(
|
|||
|
|
self.config.project_name,
|
|||
|
|
self.search_package,
|
|||
|
|
self.macro_stack,
|
|||
|
|
internal_packages,
|
|||
|
|
self.model,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def _sql_results(self) -> Dict[str, AttrDict]:
|
|||
|
|
return self.sql_results
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def load_result(self, name: str) -> Optional[AttrDict]:
|
|||
|
|
return self.sql_results.get(name)
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def store_result(
|
|||
|
|
self, name: str,
|
|||
|
|
response: Any,
|
|||
|
|
agate_table: Optional[agate.Table] = None
|
|||
|
|
) -> str:
|
|||
|
|
if agate_table is None:
|
|||
|
|
agate_table = agate_helper.empty_table()
|
|||
|
|
|
|||
|
|
self.sql_results[name] = AttrDict({
|
|||
|
|
'response': response,
|
|||
|
|
'data': agate_helper.as_matrix(agate_table),
|
|||
|
|
'table': agate_table
|
|||
|
|
})
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def store_raw_result(
|
|||
|
|
self,
|
|||
|
|
name: str,
|
|||
|
|
message=Optional[str],
|
|||
|
|
code=Optional[str],
|
|||
|
|
rows_affected=Optional[str],
|
|||
|
|
agate_table: Optional[agate.Table] = None
|
|||
|
|
) -> str:
|
|||
|
|
response = AdapterResponse(
|
|||
|
|
_message=message, code=code, rows_affected=rows_affected)
|
|||
|
|
return self.store_result(name, response, agate_table)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def validation(self):
|
|||
|
|
def validate_any(*args) -> Callable[[T], None]:
|
|||
|
|
def inner(value: T) -> None:
|
|||
|
|
for arg in args:
|
|||
|
|
if isinstance(arg, type) and isinstance(value, arg):
|
|||
|
|
return
|
|||
|
|
elif value == arg:
|
|||
|
|
return
|
|||
|
|
raise ValidationException(
|
|||
|
|
'Expected value "{}" to be one of {}'
|
|||
|
|
.format(value, ','.join(map(str, args))))
|
|||
|
|
return inner
|
|||
|
|
|
|||
|
|
return AttrDict({
|
|||
|
|
'any': validate_any,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def write(self, payload: str) -> str:
|
|||
|
|
# macros/source defs aren't 'writeable'.
|
|||
|
|
if isinstance(self.model, (ParsedMacro, ParsedSourceDefinition)):
|
|||
|
|
raise_compiler_error(
|
|||
|
|
'cannot "write" macros or sources'
|
|||
|
|
)
|
|||
|
|
self.model.build_path = self.model.write_node(
|
|||
|
|
self.config.target_path, 'run', payload
|
|||
|
|
)
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def render(self, string: str) -> str:
|
|||
|
|
return get_rendered(string, self._ctx, self.model)
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def try_or_compiler_error(
|
|||
|
|
self, message_if_exception: str, func: Callable, *args, **kwargs
|
|||
|
|
) -> Any:
|
|||
|
|
try:
|
|||
|
|
return func(*args, **kwargs)
|
|||
|
|
except Exception:
|
|||
|
|
raise_compiler_error(
|
|||
|
|
message_if_exception, self.model
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def load_agate_table(self) -> agate.Table:
|
|||
|
|
if not isinstance(self.model, (ParsedSeedNode, CompiledSeedNode)):
|
|||
|
|
raise_compiler_error(
|
|||
|
|
'can only load_agate_table for seeds (got a {})'
|
|||
|
|
.format(self.model.resource_type)
|
|||
|
|
)
|
|||
|
|
path = os.path.join(
|
|||
|
|
self.model.root_path, self.model.original_file_path
|
|||
|
|
)
|
|||
|
|
column_types = self.model.config.column_types
|
|||
|
|
try:
|
|||
|
|
table = agate_helper.from_csv(path, text_columns=column_types)
|
|||
|
|
except ValueError as e:
|
|||
|
|
raise_compiler_error(str(e))
|
|||
|
|
table.original_abspath = os.path.abspath(path)
|
|||
|
|
return table
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def ref(self) -> Callable:
|
|||
|
|
"""The most important function in dbt is `ref()`; it's impossible to
|
|||
|
|
build even moderately complex models without it. `ref()` is how you
|
|||
|
|
reference one model within another. This is a very common behavior, as
|
|||
|
|
typically models are built to be "stacked" on top of one another. Here
|
|||
|
|
is how this looks in practice:
|
|||
|
|
|
|||
|
|
> model_a.sql:
|
|||
|
|
|
|||
|
|
select *
|
|||
|
|
from public.raw_data
|
|||
|
|
|
|||
|
|
> model_b.sql:
|
|||
|
|
|
|||
|
|
select *
|
|||
|
|
from {{ref('model_a')}}
|
|||
|
|
|
|||
|
|
|
|||
|
|
`ref()` is, under the hood, actually doing two important things. First,
|
|||
|
|
it is interpolating the schema into your model file to allow you to
|
|||
|
|
change your deployment schema via configuration. Second, it is using
|
|||
|
|
these references between models to automatically build the dependency
|
|||
|
|
graph. This will enable dbt to deploy models in the correct order when
|
|||
|
|
using dbt run.
|
|||
|
|
|
|||
|
|
The `ref` function returns a Relation object.
|
|||
|
|
|
|||
|
|
## Advanced ref usage
|
|||
|
|
|
|||
|
|
There is also a two-argument variant of the `ref` function. With this
|
|||
|
|
variant, you can pass both a package name and model name to `ref` to
|
|||
|
|
avoid ambiguity. This functionality is not commonly required for
|
|||
|
|
typical dbt usage.
|
|||
|
|
|
|||
|
|
> model.sql:
|
|||
|
|
|
|||
|
|
select * from {{ ref('package_name', 'model_name') }}"
|
|||
|
|
"""
|
|||
|
|
return self.provider.ref(
|
|||
|
|
self.db_wrapper, self.model, self.config, self.manifest
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def source(self) -> Callable:
|
|||
|
|
return self.provider.source(
|
|||
|
|
self.db_wrapper, self.model, self.config, self.manifest
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextproperty('config')
|
|||
|
|
def ctx_config(self) -> Config:
|
|||
|
|
"""The `config` variable exists to handle end-user configuration for
|
|||
|
|
custom materializations. Configs like `unique_key` can be implemented
|
|||
|
|
using the `config` variable in your own materializations.
|
|||
|
|
|
|||
|
|
For example, code in the `incremental` materialization like this:
|
|||
|
|
|
|||
|
|
{% materialization incremental, default -%}
|
|||
|
|
{%- set unique_key = config.get('unique_key') -%}
|
|||
|
|
...
|
|||
|
|
|
|||
|
|
is responsible for handling model code that looks like this:
|
|||
|
|
|
|||
|
|
{{
|
|||
|
|
config(
|
|||
|
|
materialized='incremental',
|
|||
|
|
unique_key='id'
|
|||
|
|
)
|
|||
|
|
}}
|
|||
|
|
|
|||
|
|
|
|||
|
|
## config.get
|
|||
|
|
|
|||
|
|
name: The name of the configuration variable (required)
|
|||
|
|
default: The default value to use if this configuration is not provided
|
|||
|
|
(optional)
|
|||
|
|
|
|||
|
|
The `config.get` function is used to get configurations for a model
|
|||
|
|
from the end-user. Configs defined in this way are optional, and a
|
|||
|
|
default value can be provided.
|
|||
|
|
|
|||
|
|
Example usage:
|
|||
|
|
|
|||
|
|
{% materialization incremental, default -%}
|
|||
|
|
-- Example w/ no default. unique_key will be None if the user does not provide this configuration
|
|||
|
|
{%- set unique_key = config.get('unique_key') -%}
|
|||
|
|
-- Example w/ default value. Default to 'id' if 'unique_key' not provided
|
|||
|
|
{%- set unique_key = config.get('unique_key', default='id') -%}
|
|||
|
|
...
|
|||
|
|
|
|||
|
|
## config.require
|
|||
|
|
|
|||
|
|
name: The name of the configuration variable (required)
|
|||
|
|
|
|||
|
|
The `config.require` function is used to get configurations for a model
|
|||
|
|
from the end-user. Configs defined using this function are required,
|
|||
|
|
and failure to provide them will result in a compilation error.
|
|||
|
|
|
|||
|
|
Example usage:
|
|||
|
|
|
|||
|
|
{% materialization incremental, default -%}
|
|||
|
|
{%- set unique_key = config.require('unique_key') -%}
|
|||
|
|
...
|
|||
|
|
""" # noqa
|
|||
|
|
return self.provider.Config(self.model, self.context_config)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def execute(self) -> bool:
|
|||
|
|
"""`execute` is a Jinja variable that returns True when dbt is in
|
|||
|
|
"execute" mode.
|
|||
|
|
|
|||
|
|
When you execute a dbt compile or dbt run command, dbt:
|
|||
|
|
|
|||
|
|
- Reads all of the files in your project and generates a "manifest"
|
|||
|
|
comprised of models, tests, and other graph nodes present in your
|
|||
|
|
project. During this phase, dbt uses the `ref` statements it finds
|
|||
|
|
to generate the DAG for your project. *No SQL is run during this
|
|||
|
|
phase*, and `execute == False`.
|
|||
|
|
- Compiles (and runs) each node (eg. building models, or running
|
|||
|
|
tests). SQL is run during this phase, and `execute == True`.
|
|||
|
|
|
|||
|
|
Any Jinja that relies on a result being returned from the database will
|
|||
|
|
error during the parse phase. For example, this SQL will return an
|
|||
|
|
error:
|
|||
|
|
|
|||
|
|
> models/order_payment_methods.sql:
|
|||
|
|
|
|||
|
|
{% set payment_method_query %}
|
|||
|
|
select distinct
|
|||
|
|
payment_method
|
|||
|
|
from {{ ref('raw_payments') }}
|
|||
|
|
order by 1
|
|||
|
|
{% endset %}
|
|||
|
|
{% set results = run_query(relation_query) %}
|
|||
|
|
{# Return the first column #}
|
|||
|
|
{% set payment_methods = results.columns[0].values() %}
|
|||
|
|
|
|||
|
|
The error returned by dbt will look as follows:
|
|||
|
|
|
|||
|
|
Encountered an error:
|
|||
|
|
Compilation Error in model order_payment_methods (models/order_payment_methods.sql)
|
|||
|
|
'None' has no attribute 'table'
|
|||
|
|
|
|||
|
|
This is because Line #11 assumes that a table has been returned, when,
|
|||
|
|
during the parse phase, this query hasn't been run.
|
|||
|
|
|
|||
|
|
To work around this, wrap any problematic Jinja in an
|
|||
|
|
`{% if execute %}` statement:
|
|||
|
|
|
|||
|
|
> models/order_payment_methods.sql:
|
|||
|
|
|
|||
|
|
{% set payment_method_query %}
|
|||
|
|
select distinct
|
|||
|
|
payment_method
|
|||
|
|
from {{ ref('raw_payments') }}
|
|||
|
|
order by 1
|
|||
|
|
{% endset %}
|
|||
|
|
{% set results = run_query(relation_query) %}
|
|||
|
|
{% if execute %}
|
|||
|
|
{# Return the first column #}
|
|||
|
|
{% set payment_methods = results.columns[0].values() %}
|
|||
|
|
{% else %}
|
|||
|
|
{% set payment_methods = [] %}
|
|||
|
|
{% endif %}
|
|||
|
|
""" # noqa
|
|||
|
|
return self.provider.execute
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def exceptions(self) -> Dict[str, Any]:
|
|||
|
|
"""The exceptions namespace can be used to raise warnings and errors in
|
|||
|
|
dbt userspace.
|
|||
|
|
|
|||
|
|
|
|||
|
|
## raise_compiler_error
|
|||
|
|
|
|||
|
|
The `exceptions.raise_compiler_error` method will raise a compiler
|
|||
|
|
error with the provided message. This is typically only useful in
|
|||
|
|
macros or materializations when invalid arguments are provided by the
|
|||
|
|
calling model. Note that throwing an exception will cause a model to
|
|||
|
|
fail, so please use this variable with care!
|
|||
|
|
|
|||
|
|
Example usage:
|
|||
|
|
|
|||
|
|
> exceptions.sql:
|
|||
|
|
|
|||
|
|
{% if number < 0 or number > 100 %}
|
|||
|
|
{{ exceptions.raise_compiler_error("Invalid `number`. Got: " ~ number) }}
|
|||
|
|
{% endif %}
|
|||
|
|
|
|||
|
|
## warn
|
|||
|
|
|
|||
|
|
The `exceptions.warn` method will raise a compiler warning with the
|
|||
|
|
provided message. If the `--warn-error` flag is provided to dbt, then
|
|||
|
|
this warning will be elevated to an exception, which is raised.
|
|||
|
|
|
|||
|
|
Example usage:
|
|||
|
|
|
|||
|
|
> warn.sql:
|
|||
|
|
|
|||
|
|
{% if number < 0 or number > 100 %}
|
|||
|
|
{% do exceptions.warn("Invalid `number`. Got: " ~ number) %}
|
|||
|
|
{% endif %}
|
|||
|
|
""" # noqa
|
|||
|
|
return wrapped_exports(self.model)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def database(self) -> str:
|
|||
|
|
return self.config.credentials.database
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def schema(self) -> str:
|
|||
|
|
return self.config.credentials.schema
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def var(self) -> ModelConfiguredVar:
|
|||
|
|
return self.provider.Var(
|
|||
|
|
context=self._ctx,
|
|||
|
|
config=self.config,
|
|||
|
|
node=self.model,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextproperty('adapter')
|
|||
|
|
def ctx_adapter(self) -> BaseDatabaseWrapper:
|
|||
|
|
"""`adapter` is a wrapper around the internal database adapter used by
|
|||
|
|
dbt. It allows users to make calls to the database in their dbt models.
|
|||
|
|
The adapter methods will be translated into specific SQL statements
|
|||
|
|
depending on the type of adapter your project is using.
|
|||
|
|
"""
|
|||
|
|
return self.db_wrapper
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def api(self) -> Dict[str, Any]:
|
|||
|
|
return {
|
|||
|
|
'Relation': self.db_wrapper.Relation,
|
|||
|
|
'Column': self.adapter.Column,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def column(self) -> Type[Column]:
|
|||
|
|
return self.adapter.Column
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def env(self) -> Dict[str, Any]:
|
|||
|
|
return self.target
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def graph(self) -> Dict[str, Any]:
|
|||
|
|
"""The `graph` context variable contains information about the nodes in
|
|||
|
|
your dbt project. Models, sources, tests, and snapshots are all
|
|||
|
|
examples of nodes in dbt projects.
|
|||
|
|
|
|||
|
|
## The graph context variable
|
|||
|
|
|
|||
|
|
The graph context variable is a dictionary which maps node ids onto dictionary representations of those nodes. A simplified example might look like:
|
|||
|
|
|
|||
|
|
{
|
|||
|
|
"model.project_name.model_name": {
|
|||
|
|
"config": {"materialzed": "table", "sort": "id"},
|
|||
|
|
"tags": ["abc", "123"],
|
|||
|
|
"path": "models/path/to/model_name.sql",
|
|||
|
|
...
|
|||
|
|
},
|
|||
|
|
"source.project_name.source_name": {
|
|||
|
|
"path": "models/path/to/schema.yml",
|
|||
|
|
"columns": {
|
|||
|
|
"id": { .... },
|
|||
|
|
"first_name": { .... },
|
|||
|
|
},
|
|||
|
|
...
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
The exact contract for these model and source nodes is not currently
|
|||
|
|
documented, but that will change in the future.
|
|||
|
|
|
|||
|
|
## Accessing models
|
|||
|
|
|
|||
|
|
The `model` entries in the `graph` dictionary will be incomplete or
|
|||
|
|
incorrect during parsing. If accessing the models in your project via
|
|||
|
|
the `graph` variable, be sure to use the `execute` flag to ensure that
|
|||
|
|
this code only executes at run-time and not at parse-time. Do not use
|
|||
|
|
the `graph` variable to build you DAG, as the resulting dbt behavior
|
|||
|
|
will be undefined and likely incorrect.
|
|||
|
|
|
|||
|
|
Example usage:
|
|||
|
|
|
|||
|
|
> graph-usage.sql:
|
|||
|
|
|
|||
|
|
/*
|
|||
|
|
Print information about all of the models in the Snowplow package
|
|||
|
|
*/
|
|||
|
|
{% if execute %}
|
|||
|
|
{% for node in graph.nodes.values()
|
|||
|
|
| selectattr("resource_type", "equalto", "model")
|
|||
|
|
| selectattr("package_name", "equalto", "snowplow") %}
|
|||
|
|
|
|||
|
|
{% do log(node.unique_id ~ ", materialized: " ~ node.config.materialized, info=true) %}
|
|||
|
|
|
|||
|
|
{% endfor %}
|
|||
|
|
{% endif %}
|
|||
|
|
/*
|
|||
|
|
Example output
|
|||
|
|
---------------------------------------------------------------
|
|||
|
|
model.snowplow.snowplow_id_map, materialized: incremental
|
|||
|
|
model.snowplow.snowplow_page_views, materialized: incremental
|
|||
|
|
model.snowplow.snowplow_web_events, materialized: incremental
|
|||
|
|
model.snowplow.snowplow_web_page_context, materialized: table
|
|||
|
|
model.snowplow.snowplow_web_events_scroll_depth, materialized: incremental
|
|||
|
|
model.snowplow.snowplow_web_events_time, materialized: incremental
|
|||
|
|
model.snowplow.snowplow_web_events_internal_fixed, materialized: ephemeral
|
|||
|
|
model.snowplow.snowplow_base_web_page_context, materialized: ephemeral
|
|||
|
|
model.snowplow.snowplow_base_events, materialized: ephemeral
|
|||
|
|
model.snowplow.snowplow_sessions_tmp, materialized: incremental
|
|||
|
|
model.snowplow.snowplow_sessions, materialized: table
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
## Accessing sources
|
|||
|
|
|
|||
|
|
To access the sources in your dbt project programatically, use the "sources" attribute.
|
|||
|
|
|
|||
|
|
Example usage:
|
|||
|
|
|
|||
|
|
> models/events_unioned.sql
|
|||
|
|
|
|||
|
|
/*
|
|||
|
|
Union all of the Snowplow sources defined in the project
|
|||
|
|
which begin with the string "event_"
|
|||
|
|
*/
|
|||
|
|
{% set sources = [] -%}
|
|||
|
|
{% for node in graph.sources.values() -%}
|
|||
|
|
{%- if node.name.startswith('event_') and node.source_name == 'snowplow' -%}
|
|||
|
|
{%- do sources.append(source(node.source_name, node.name)) -%}
|
|||
|
|
{%- endif -%}
|
|||
|
|
{%- endfor %}
|
|||
|
|
select * from (
|
|||
|
|
{%- for source in sources %}
|
|||
|
|
{{ source }} {% if not loop.last %} union all {% endif %}
|
|||
|
|
{% endfor %}
|
|||
|
|
)
|
|||
|
|
/*
|
|||
|
|
Example compiled SQL
|
|||
|
|
---------------------------------------------------------------
|
|||
|
|
select * from (
|
|||
|
|
select * from raw.snowplow.event_add_to_cart union all
|
|||
|
|
select * from raw.snowplow.event_remove_from_cart union all
|
|||
|
|
select * from raw.snowplow.event_checkout
|
|||
|
|
)
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
""" # noqa
|
|||
|
|
return self.manifest.flat_graph
|
|||
|
|
|
|||
|
|
@contextproperty('model')
|
|||
|
|
def ctx_model(self) -> Dict[str, Any]:
|
|||
|
|
return self.model.to_dict(omit_none=True)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def pre_hooks(self) -> Optional[List[Dict[str, Any]]]:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def post_hooks(self) -> Optional[List[Dict[str, Any]]]:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def sql(self) -> Optional[str]:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def sql_now(self) -> str:
|
|||
|
|
return self.adapter.date_function()
|
|||
|
|
|
|||
|
|
@contextmember
|
|||
|
|
def adapter_macro(self, name: str, *args, **kwargs):
|
|||
|
|
"""Find the most appropriate macro for the name, considering the
|
|||
|
|
adapter type currently in use, and call that with the given arguments.
|
|||
|
|
|
|||
|
|
If the name has a `.` in it, the first section before the `.` is
|
|||
|
|
interpreted as a package name, and the remainder as a macro name.
|
|||
|
|
|
|||
|
|
If no adapter is found, raise a compiler exception. If an invalid
|
|||
|
|
package name is specified, raise a compiler exception.
|
|||
|
|
|
|||
|
|
|
|||
|
|
Some examples:
|
|||
|
|
|
|||
|
|
{# dbt will call this macro by name, providing any arguments #}
|
|||
|
|
{% macro create_table_as(temporary, relation, sql) -%}
|
|||
|
|
|
|||
|
|
{# dbt will dispatch the macro call to the relevant macro #}
|
|||
|
|
{{ adapter_macro('create_table_as', temporary, relation, sql) }}
|
|||
|
|
{%- endmacro %}
|
|||
|
|
|
|||
|
|
|
|||
|
|
{#
|
|||
|
|
If no macro matches the specified adapter, "default" will be
|
|||
|
|
used
|
|||
|
|
#}
|
|||
|
|
{% macro default__create_table_as(temporary, relation, sql) -%}
|
|||
|
|
...
|
|||
|
|
{%- endmacro %}
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
{# Example which defines special logic for Redshift #}
|
|||
|
|
{% macro redshift__create_table_as(temporary, relation, sql) -%}
|
|||
|
|
...
|
|||
|
|
{%- endmacro %}
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
{# Example which defines special logic for BigQuery #}
|
|||
|
|
{% macro bigquery__create_table_as(temporary, relation, sql) -%}
|
|||
|
|
...
|
|||
|
|
{%- endmacro %}
|
|||
|
|
"""
|
|||
|
|
deprecations.warn('adapter-macro', macro_name=name)
|
|||
|
|
original_name = name
|
|||
|
|
package_name = None
|
|||
|
|
if '.' in name:
|
|||
|
|
package_name, name = name.split('.', 1)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
macro = self.db_wrapper.dispatch(
|
|||
|
|
macro_name=name, macro_namespace=package_name
|
|||
|
|
)
|
|||
|
|
except CompilationException as exc:
|
|||
|
|
raise CompilationException(
|
|||
|
|
f'In adapter_macro: {exc.msg}\n'
|
|||
|
|
f" Original name: '{original_name}'",
|
|||
|
|
node=self.model
|
|||
|
|
) from exc
|
|||
|
|
return macro(*args, **kwargs)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class MacroContext(ProviderContext):
|
|||
|
|
"""Internally, macros can be executed like nodes, with some restrictions:
|
|||
|
|
|
|||
|
|
- they don't have have all values available that nodes do:
|
|||
|
|
- 'this', 'pre_hooks', 'post_hooks', and 'sql' are missing
|
|||
|
|
- 'schema' does not use any 'model' information
|
|||
|
|
- they can't be configured with config() directives
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
model: ParsedMacro,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
provider: Provider,
|
|||
|
|
search_package: Optional[str],
|
|||
|
|
) -> None:
|
|||
|
|
super().__init__(model, config, manifest, provider, None)
|
|||
|
|
# override the model-based package with the given one
|
|||
|
|
if search_package is None:
|
|||
|
|
# if the search package name isn't specified, use the root project
|
|||
|
|
self._search_package = config.project_name
|
|||
|
|
else:
|
|||
|
|
self._search_package = search_package
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ModelContext(ProviderContext):
|
|||
|
|
model: ManifestNode
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def pre_hooks(self) -> List[Dict[str, Any]]:
|
|||
|
|
if self.model.resource_type in [NodeType.Source, NodeType.Test]:
|
|||
|
|
return []
|
|||
|
|
return [
|
|||
|
|
h.to_dict(omit_none=True) for h in self.model.config.pre_hook
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def post_hooks(self) -> List[Dict[str, Any]]:
|
|||
|
|
if self.model.resource_type in [NodeType.Source, NodeType.Test]:
|
|||
|
|
return []
|
|||
|
|
return [
|
|||
|
|
h.to_dict(omit_none=True) for h in self.model.config.post_hook
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def sql(self) -> Optional[str]:
|
|||
|
|
if getattr(self.model, 'extra_ctes_injected', None):
|
|||
|
|
return self.model.compiled_sql
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def database(self) -> str:
|
|||
|
|
return getattr(
|
|||
|
|
self.model, 'database', self.config.credentials.database
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def schema(self) -> str:
|
|||
|
|
return getattr(
|
|||
|
|
self.model, 'schema', self.config.credentials.schema
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@contextproperty
|
|||
|
|
def this(self) -> Optional[RelationProxy]:
|
|||
|
|
"""`this` makes available schema information about the currently
|
|||
|
|
executing model. It's is useful in any context in which you need to
|
|||
|
|
write code that references the current model, for example when defining
|
|||
|
|
a `sql_where` clause for an incremental model and for writing pre- and
|
|||
|
|
post-model hooks that operate on the model in some way. Developers have
|
|||
|
|
options for how to use `this`:
|
|||
|
|
|
|||
|
|
|------------------|------------------|
|
|||
|
|
| dbt Model Syntax | Output |
|
|||
|
|
|------------------|------------------|
|
|||
|
|
| {{this}} | "schema"."table" |
|
|||
|
|
|------------------|------------------|
|
|||
|
|
| {{this.schema}} | schema |
|
|||
|
|
|------------------|------------------|
|
|||
|
|
| {{this.table}} | table |
|
|||
|
|
|------------------|------------------|
|
|||
|
|
| {{this.name}} | table |
|
|||
|
|
|------------------|------------------|
|
|||
|
|
|
|||
|
|
Here's an example of how to use `this` in `dbt_project.yml` to grant
|
|||
|
|
select rights on a table to a different db user.
|
|||
|
|
|
|||
|
|
> example.yml:
|
|||
|
|
|
|||
|
|
models:
|
|||
|
|
project-name:
|
|||
|
|
post-hook:
|
|||
|
|
- "grant select on {{ this }} to db_reader"
|
|||
|
|
"""
|
|||
|
|
if self.model.resource_type == NodeType.Operation:
|
|||
|
|
return None
|
|||
|
|
return self.db_wrapper.Relation.create_from(self.config, self.model)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# This is called by '_context_for', used in 'render_with_context'
|
|||
|
|
def generate_parser_model(
|
|||
|
|
model: ManifestNode,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
context_config: ContextConfig,
|
|||
|
|
) -> Dict[str, Any]:
|
|||
|
|
# The __init__ method of ModelContext also initializes
|
|||
|
|
# a ManifestContext object which creates a MacroNamespaceBuilder
|
|||
|
|
# which adds every macro in the Manifest.
|
|||
|
|
ctx = ModelContext(
|
|||
|
|
model, config, manifest, ParseProvider(), context_config
|
|||
|
|
)
|
|||
|
|
# The 'to_dict' method in ManifestContext moves all of the macro names
|
|||
|
|
# in the macro 'namespace' up to top level keys
|
|||
|
|
return ctx.to_dict()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_generate_component_name_macro(
|
|||
|
|
macro: ParsedMacro,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
) -> Dict[str, Any]:
|
|||
|
|
ctx = MacroContext(
|
|||
|
|
macro, config, manifest, GenerateNameProvider(), None
|
|||
|
|
)
|
|||
|
|
return ctx.to_dict()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_runtime_model(
|
|||
|
|
model: ManifestNode,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
) -> Dict[str, Any]:
|
|||
|
|
ctx = ModelContext(
|
|||
|
|
model, config, manifest, RuntimeProvider(), None
|
|||
|
|
)
|
|||
|
|
return ctx.to_dict()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_runtime_macro(
|
|||
|
|
macro: ParsedMacro,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
package_name: Optional[str],
|
|||
|
|
) -> Dict[str, Any]:
|
|||
|
|
ctx = MacroContext(
|
|||
|
|
macro, config, manifest, OperationProvider(), package_name
|
|||
|
|
)
|
|||
|
|
return ctx.to_dict()
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ExposureRefResolver(BaseResolver):
|
|||
|
|
def __call__(self, *args) -> str:
|
|||
|
|
if len(args) not in (1, 2):
|
|||
|
|
ref_invalid_args(self.model, args)
|
|||
|
|
self.model.refs.append(list(args))
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ExposureSourceResolver(BaseResolver):
|
|||
|
|
def __call__(self, *args) -> str:
|
|||
|
|
if len(args) != 2:
|
|||
|
|
raise_compiler_error(
|
|||
|
|
f"source() takes exactly two arguments ({len(args)} given)",
|
|||
|
|
self.model
|
|||
|
|
)
|
|||
|
|
self.model.sources.append(list(args))
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_parse_exposure(
|
|||
|
|
exposure: ParsedExposure,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
package_name: str,
|
|||
|
|
) -> Dict[str, Any]:
|
|||
|
|
project = config.load_dependencies()[package_name]
|
|||
|
|
return {
|
|||
|
|
'ref': ExposureRefResolver(
|
|||
|
|
None,
|
|||
|
|
exposure,
|
|||
|
|
project,
|
|||
|
|
manifest,
|
|||
|
|
),
|
|||
|
|
'source': ExposureSourceResolver(
|
|||
|
|
None,
|
|||
|
|
exposure,
|
|||
|
|
project,
|
|||
|
|
manifest,
|
|||
|
|
)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# This class is currently used by the schema parser in order
|
|||
|
|
# to limit the number of macros in the context by using
|
|||
|
|
# the TestMacroNamespace
|
|||
|
|
class TestContext(ProviderContext):
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
model,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
provider: Provider,
|
|||
|
|
context_config: Optional[ContextConfig],
|
|||
|
|
macro_resolver: MacroResolver,
|
|||
|
|
) -> None:
|
|||
|
|
# this must be before super init so that macro_resolver exists for
|
|||
|
|
# build_namespace
|
|||
|
|
self.macro_resolver = macro_resolver
|
|||
|
|
self.thread_ctx = MacroStack()
|
|||
|
|
super().__init__(model, config, manifest, provider, context_config)
|
|||
|
|
self._build_test_namespace()
|
|||
|
|
# We need to rebuild this because it's already been built by
|
|||
|
|
# the ProviderContext with the wrong namespace.
|
|||
|
|
self.db_wrapper = self.provider.DatabaseWrapper(
|
|||
|
|
self.adapter, self.namespace
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def _build_namespace(self):
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
# this overrides _build_namespace in ManifestContext which provides a
|
|||
|
|
# complete namespace of all macros to only specify macros in the depends_on
|
|||
|
|
# This only provides a namespace with macros in the test node
|
|||
|
|
# 'depends_on.macros' by using the TestMacroNamespace
|
|||
|
|
def _build_test_namespace(self):
|
|||
|
|
depends_on_macros = []
|
|||
|
|
# all generic tests use a macro named 'get_where_subquery' to wrap 'model' arg
|
|||
|
|
# see generic_test_builders.build_model_str
|
|||
|
|
get_where_subquery = self.macro_resolver.macros_by_name.get('get_where_subquery')
|
|||
|
|
if get_where_subquery:
|
|||
|
|
depends_on_macros.append(get_where_subquery.unique_id)
|
|||
|
|
if self.model.depends_on and self.model.depends_on.macros:
|
|||
|
|
depends_on_macros.extend(self.model.depends_on.macros)
|
|||
|
|
lookup_macros = depends_on_macros.copy()
|
|||
|
|
for macro_unique_id in lookup_macros:
|
|||
|
|
lookup_macro = self.macro_resolver.macros.get(macro_unique_id)
|
|||
|
|
if lookup_macro:
|
|||
|
|
depends_on_macros.extend(lookup_macro.depends_on.macros)
|
|||
|
|
|
|||
|
|
macro_namespace = TestMacroNamespace(
|
|||
|
|
self.macro_resolver, self._ctx, self.model, self.thread_ctx,
|
|||
|
|
depends_on_macros
|
|||
|
|
)
|
|||
|
|
self.namespace = macro_namespace
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_test_context(
|
|||
|
|
model: ManifestNode,
|
|||
|
|
config: RuntimeConfig,
|
|||
|
|
manifest: Manifest,
|
|||
|
|
context_config: ContextConfig,
|
|||
|
|
macro_resolver: MacroResolver
|
|||
|
|
) -> Dict[str, Any]:
|
|||
|
|
ctx = TestContext(
|
|||
|
|
model, config, manifest, ParseProvider(), context_config,
|
|||
|
|
macro_resolver
|
|||
|
|
)
|
|||
|
|
# The 'to_dict' method in ManifestContext moves all of the macro names
|
|||
|
|
# in the macro 'namespace' up to top level keys
|
|||
|
|
return ctx.to_dict()
|