448 lines
16 KiB
Python
448 lines
16 KiB
Python
|
from dataclasses import dataclass
|
||
|
|
from typing import Any, Dict, Optional, Tuple
|
||
|
|
import os
|
||
|
|
|
||
|
|
from dbt.dataclass_schema import ValidationError
|
||
|
|
|
||
|
|
from dbt.clients.system import load_file_contents
|
||
|
|
from dbt.clients.yaml_helper import load_yaml_text
|
||
|
|
from dbt.contracts.connection import Credentials, HasCredentials
|
||
|
|
from dbt.contracts.project import ProfileConfig, UserConfig
|
||
|
|
from dbt.exceptions import CompilationException
|
||
|
|
from dbt.exceptions import DbtProfileError
|
||
|
|
from dbt.exceptions import DbtProjectError
|
||
|
|
from dbt.exceptions import ValidationException
|
||
|
|
from dbt.exceptions import RuntimeException
|
||
|
|
from dbt.exceptions import validator_error_message
|
||
|
|
from dbt.logger import GLOBAL_LOGGER as logger
|
||
|
|
from dbt.utils import coerce_dict_str
|
||
|
|
|
||
|
|
from .renderer import ProfileRenderer
|
||
|
|
|
||
|
|
DEFAULT_THREADS = 1
|
||
|
|
DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt')
|
||
|
|
PROFILES_DIR = os.path.expanduser(
|
||
|
|
os.getenv('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR)
|
||
|
|
)
|
||
|
|
|
||
|
|
INVALID_PROFILE_MESSAGE = """
|
||
|
|
dbt encountered an error while trying to read your profiles.yml file.
|
||
|
|
|
||
|
|
{error_string}
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
NO_SUPPLIED_PROFILE_ERROR = """\
|
||
|
|
dbt cannot run because no profile was specified for this dbt project.
|
||
|
|
To specify a profile for this project, add a line like the this to
|
||
|
|
your dbt_project.yml file:
|
||
|
|
|
||
|
|
profile: [profile name]
|
||
|
|
|
||
|
|
Here, [profile name] should be replaced with a profile name
|
||
|
|
defined in your profiles.yml file. You can find profiles.yml here:
|
||
|
|
|
||
|
|
{profiles_file}/profiles.yml
|
||
|
|
""".format(profiles_file=PROFILES_DIR)
|
||
|
|
|
||
|
|
|
||
|
|
def read_profile(profiles_dir: str) -> Dict[str, Any]:
|
||
|
|
path = os.path.join(profiles_dir, 'profiles.yml')
|
||
|
|
|
||
|
|
contents = None
|
||
|
|
if os.path.isfile(path):
|
||
|
|
try:
|
||
|
|
contents = load_file_contents(path, strip=False)
|
||
|
|
yaml_content = load_yaml_text(contents)
|
||
|
|
if not yaml_content:
|
||
|
|
msg = f'The profiles.yml file at {path} is empty'
|
||
|
|
raise DbtProfileError(
|
||
|
|
INVALID_PROFILE_MESSAGE.format(
|
||
|
|
error_string=msg
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return yaml_content
|
||
|
|
except ValidationException as e:
|
||
|
|
msg = INVALID_PROFILE_MESSAGE.format(error_string=e)
|
||
|
|
raise ValidationException(msg) from e
|
||
|
|
|
||
|
|
return {}
|
||
|
|
|
||
|
|
|
||
|
|
def read_user_config(directory: str) -> UserConfig:
|
||
|
|
try:
|
||
|
|
profile = read_profile(directory)
|
||
|
|
if profile:
|
||
|
|
user_cfg = coerce_dict_str(profile.get('config', {}))
|
||
|
|
if user_cfg is not None:
|
||
|
|
UserConfig.validate(user_cfg)
|
||
|
|
return UserConfig.from_dict(user_cfg)
|
||
|
|
except (RuntimeException, ValidationError):
|
||
|
|
pass
|
||
|
|
return UserConfig()
|
||
|
|
|
||
|
|
|
||
|
|
# The Profile class is included in RuntimeConfig, so any attribute
|
||
|
|
# additions must also be set where the RuntimeConfig class is created
|
||
|
|
# `init=False` is a workaround for https://bugs.python.org/issue45081
|
||
|
|
@dataclass(init=False)
|
||
|
|
class Profile(HasCredentials):
|
||
|
|
profile_name: str
|
||
|
|
target_name: str
|
||
|
|
config: UserConfig
|
||
|
|
threads: int
|
||
|
|
credentials: Credentials
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
profile_name: str,
|
||
|
|
target_name: str,
|
||
|
|
config: UserConfig,
|
||
|
|
threads: int,
|
||
|
|
credentials: Credentials
|
||
|
|
):
|
||
|
|
"""Explicitly defining `__init__` to work around bug in Python 3.9.7
|
||
|
|
https://bugs.python.org/issue45081
|
||
|
|
"""
|
||
|
|
self.profile_name = profile_name
|
||
|
|
self.target_name = target_name
|
||
|
|
self.config = config
|
||
|
|
self.threads = threads
|
||
|
|
self.credentials = credentials
|
||
|
|
|
||
|
|
def to_profile_info(
|
||
|
|
self, serialize_credentials: bool = False
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
"""Unlike to_project_config, this dict is not a mirror of any existing
|
||
|
|
on-disk data structure. It's used when creating a new profile from an
|
||
|
|
existing one.
|
||
|
|
|
||
|
|
:param serialize_credentials bool: If True, serialize the credentials.
|
||
|
|
Otherwise, the Credentials object will be copied.
|
||
|
|
:returns dict: The serialized profile.
|
||
|
|
"""
|
||
|
|
result = {
|
||
|
|
'profile_name': self.profile_name,
|
||
|
|
'target_name': self.target_name,
|
||
|
|
'config': self.config,
|
||
|
|
'threads': self.threads,
|
||
|
|
'credentials': self.credentials,
|
||
|
|
}
|
||
|
|
if serialize_credentials:
|
||
|
|
result['config'] = self.config.to_dict(omit_none=True)
|
||
|
|
result['credentials'] = self.credentials.to_dict(omit_none=True)
|
||
|
|
return result
|
||
|
|
|
||
|
|
def to_target_dict(self) -> Dict[str, Any]:
|
||
|
|
target = dict(
|
||
|
|
self.credentials.connection_info(with_aliases=True)
|
||
|
|
)
|
||
|
|
target.update({
|
||
|
|
'type': self.credentials.type,
|
||
|
|
'threads': self.threads,
|
||
|
|
'name': self.target_name,
|
||
|
|
'target_name': self.target_name,
|
||
|
|
'profile_name': self.profile_name,
|
||
|
|
'config': self.config.to_dict(omit_none=True),
|
||
|
|
})
|
||
|
|
return target
|
||
|
|
|
||
|
|
def __eq__(self, other: object) -> bool:
|
||
|
|
if not (isinstance(other, self.__class__) and
|
||
|
|
isinstance(self, other.__class__)):
|
||
|
|
return NotImplemented
|
||
|
|
return self.to_profile_info() == other.to_profile_info()
|
||
|
|
|
||
|
|
def validate(self):
|
||
|
|
try:
|
||
|
|
if self.credentials:
|
||
|
|
dct = self.credentials.to_dict(omit_none=True)
|
||
|
|
self.credentials.validate(dct)
|
||
|
|
dct = self.to_profile_info(serialize_credentials=True)
|
||
|
|
ProfileConfig.validate(dct)
|
||
|
|
except ValidationError as exc:
|
||
|
|
raise DbtProfileError(validator_error_message(exc)) from exc
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _credentials_from_profile(
|
||
|
|
profile: Dict[str, Any], profile_name: str, target_name: str
|
||
|
|
) -> Credentials:
|
||
|
|
# avoid an import cycle
|
||
|
|
from dbt.adapters.factory import load_plugin
|
||
|
|
# credentials carry their 'type' in their actual type, not their
|
||
|
|
# attributes. We do want this in order to pick our Credentials class.
|
||
|
|
if 'type' not in profile:
|
||
|
|
raise DbtProfileError(
|
||
|
|
'required field "type" not found in profile {} and target {}'
|
||
|
|
.format(profile_name, target_name))
|
||
|
|
|
||
|
|
typename = profile.pop('type')
|
||
|
|
try:
|
||
|
|
cls = load_plugin(typename)
|
||
|
|
data = cls.translate_aliases(profile)
|
||
|
|
cls.validate(data)
|
||
|
|
credentials = cls.from_dict(data)
|
||
|
|
except (RuntimeException, ValidationError) as e:
|
||
|
|
msg = str(e) if isinstance(e, RuntimeException) else e.message
|
||
|
|
raise DbtProfileError(
|
||
|
|
'Credentials in profile "{}", target "{}" invalid: {}'
|
||
|
|
.format(profile_name, target_name, msg)
|
||
|
|
) from e
|
||
|
|
|
||
|
|
return credentials
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def pick_profile_name(
|
||
|
|
args_profile_name: Optional[str],
|
||
|
|
project_profile_name: Optional[str] = None,
|
||
|
|
) -> str:
|
||
|
|
profile_name = project_profile_name
|
||
|
|
if args_profile_name is not None:
|
||
|
|
profile_name = args_profile_name
|
||
|
|
if profile_name is None:
|
||
|
|
raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR)
|
||
|
|
return profile_name
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _get_profile_data(
|
||
|
|
profile: Dict[str, Any], profile_name: str, target_name: str
|
||
|
|
) -> Dict[str, Any]:
|
||
|
|
if 'outputs' not in profile:
|
||
|
|
raise DbtProfileError(
|
||
|
|
"outputs not specified in profile '{}'".format(profile_name)
|
||
|
|
)
|
||
|
|
outputs = profile['outputs']
|
||
|
|
|
||
|
|
if target_name not in outputs:
|
||
|
|
outputs = '\n'.join(' - {}'.format(output)
|
||
|
|
for output in outputs)
|
||
|
|
msg = ("The profile '{}' does not have a target named '{}'. The "
|
||
|
|
"valid target names for this profile are:\n{}"
|
||
|
|
.format(profile_name, target_name, outputs))
|
||
|
|
raise DbtProfileError(msg, result_type='invalid_target')
|
||
|
|
profile_data = outputs[target_name]
|
||
|
|
|
||
|
|
if not isinstance(profile_data, dict):
|
||
|
|
msg = (
|
||
|
|
f"output '{target_name}' of profile '{profile_name}' is "
|
||
|
|
f"misconfigured in profiles.yml"
|
||
|
|
)
|
||
|
|
raise DbtProfileError(msg, result_type='invalid_target')
|
||
|
|
|
||
|
|
return profile_data
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_credentials(
|
||
|
|
cls,
|
||
|
|
credentials: Credentials,
|
||
|
|
threads: int,
|
||
|
|
profile_name: str,
|
||
|
|
target_name: str,
|
||
|
|
user_cfg: Optional[Dict[str, Any]] = None
|
||
|
|
) -> 'Profile':
|
||
|
|
"""Create a profile from an existing set of Credentials and the
|
||
|
|
remaining information.
|
||
|
|
|
||
|
|
:param credentials: The credentials dict for this profile.
|
||
|
|
:param threads: The number of threads to use for connections.
|
||
|
|
:param profile_name: The profile name used for this profile.
|
||
|
|
:param target_name: The target name used for this profile.
|
||
|
|
:param user_cfg: The user-level config block from the
|
||
|
|
raw profiles, if specified.
|
||
|
|
:raises DbtProfileError: If the profile is invalid.
|
||
|
|
:returns: The new Profile object.
|
||
|
|
"""
|
||
|
|
if user_cfg is None:
|
||
|
|
user_cfg = {}
|
||
|
|
UserConfig.validate(user_cfg)
|
||
|
|
config = UserConfig.from_dict(user_cfg)
|
||
|
|
|
||
|
|
profile = cls(
|
||
|
|
profile_name=profile_name,
|
||
|
|
target_name=target_name,
|
||
|
|
config=config,
|
||
|
|
threads=threads,
|
||
|
|
credentials=credentials
|
||
|
|
)
|
||
|
|
profile.validate()
|
||
|
|
return profile
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def render_profile(
|
||
|
|
cls,
|
||
|
|
raw_profile: Dict[str, Any],
|
||
|
|
profile_name: str,
|
||
|
|
target_override: Optional[str],
|
||
|
|
renderer: ProfileRenderer,
|
||
|
|
) -> Tuple[str, Dict[str, Any]]:
|
||
|
|
"""This is a containment zone for the hateful way we're rendering
|
||
|
|
profiles.
|
||
|
|
"""
|
||
|
|
# rendering profiles is a bit complex. Two constraints cause trouble:
|
||
|
|
# 1) users should be able to use environment/cli variables to specify
|
||
|
|
# the target in their profile.
|
||
|
|
# 2) Missing environment/cli variables in profiles/targets that don't
|
||
|
|
# end up getting selected should not cause errors.
|
||
|
|
# so first we'll just render the target name, then we use that rendered
|
||
|
|
# name to extract a profile that we can render.
|
||
|
|
if target_override is not None:
|
||
|
|
target_name = target_override
|
||
|
|
elif 'target' in raw_profile:
|
||
|
|
# render the target if it was parsed from yaml
|
||
|
|
target_name = renderer.render_value(raw_profile['target'])
|
||
|
|
else:
|
||
|
|
target_name = 'default'
|
||
|
|
logger.debug(
|
||
|
|
"target not specified in profile '{}', using '{}'"
|
||
|
|
.format(profile_name, target_name)
|
||
|
|
)
|
||
|
|
|
||
|
|
raw_profile_data = cls._get_profile_data(
|
||
|
|
raw_profile, profile_name, target_name
|
||
|
|
)
|
||
|
|
|
||
|
|
try:
|
||
|
|
profile_data = renderer.render_data(raw_profile_data)
|
||
|
|
except CompilationException as exc:
|
||
|
|
raise DbtProfileError(str(exc)) from exc
|
||
|
|
return target_name, profile_data
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_raw_profile_info(
|
||
|
|
cls,
|
||
|
|
raw_profile: Dict[str, Any],
|
||
|
|
profile_name: str,
|
||
|
|
renderer: ProfileRenderer,
|
||
|
|
user_cfg: Optional[Dict[str, Any]] = None,
|
||
|
|
target_override: Optional[str] = None,
|
||
|
|
threads_override: Optional[int] = None,
|
||
|
|
) -> 'Profile':
|
||
|
|
"""Create a profile from its raw profile information.
|
||
|
|
|
||
|
|
(this is an intermediate step, mostly useful for unit testing)
|
||
|
|
|
||
|
|
:param raw_profile: The profile data for a single profile, from
|
||
|
|
disk as yaml and its values rendered with jinja.
|
||
|
|
:param profile_name: The profile name used.
|
||
|
|
:param renderer: The config renderer.
|
||
|
|
:param user_cfg: The global config for the user, if it
|
||
|
|
was present.
|
||
|
|
:param target_override: The target to use, if provided on
|
||
|
|
the command line.
|
||
|
|
:param threads_override: The thread count to use, if
|
||
|
|
provided on the command line.
|
||
|
|
:raises DbtProfileError: If the profile is invalid or missing, or the
|
||
|
|
target could not be found
|
||
|
|
:returns: The new Profile object.
|
||
|
|
"""
|
||
|
|
# user_cfg is not rendered.
|
||
|
|
if user_cfg is None:
|
||
|
|
user_cfg = raw_profile.get('config')
|
||
|
|
# TODO: should it be, and the values coerced to bool?
|
||
|
|
target_name, profile_data = cls.render_profile(
|
||
|
|
raw_profile, profile_name, target_override, renderer
|
||
|
|
)
|
||
|
|
|
||
|
|
# valid connections never include the number of threads, but it's
|
||
|
|
# stored on a per-connection level in the raw configs
|
||
|
|
threads = profile_data.pop('threads', DEFAULT_THREADS)
|
||
|
|
if threads_override is not None:
|
||
|
|
threads = threads_override
|
||
|
|
|
||
|
|
credentials: Credentials = cls._credentials_from_profile(
|
||
|
|
profile_data, profile_name, target_name
|
||
|
|
)
|
||
|
|
|
||
|
|
return cls.from_credentials(
|
||
|
|
credentials=credentials,
|
||
|
|
profile_name=profile_name,
|
||
|
|
target_name=target_name,
|
||
|
|
threads=threads,
|
||
|
|
user_cfg=user_cfg
|
||
|
|
)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_raw_profiles(
|
||
|
|
cls,
|
||
|
|
raw_profiles: Dict[str, Any],
|
||
|
|
profile_name: str,
|
||
|
|
renderer: ProfileRenderer,
|
||
|
|
target_override: Optional[str] = None,
|
||
|
|
threads_override: Optional[int] = None,
|
||
|
|
) -> 'Profile':
|
||
|
|
"""
|
||
|
|
:param raw_profiles: The profile data, from disk as yaml.
|
||
|
|
:param profile_name: The profile name to use.
|
||
|
|
:param renderer: The config renderer.
|
||
|
|
:param target_override: The target to use, if provided on the command
|
||
|
|
line.
|
||
|
|
:param threads_override: The thread count to use, if provided on the
|
||
|
|
command line.
|
||
|
|
:raises DbtProjectError: If there is no profile name specified in the
|
||
|
|
project or the command line arguments
|
||
|
|
:raises DbtProfileError: If the profile is invalid or missing, or the
|
||
|
|
target could not be found
|
||
|
|
:returns: The new Profile object.
|
||
|
|
"""
|
||
|
|
if profile_name not in raw_profiles:
|
||
|
|
raise DbtProjectError(
|
||
|
|
"Could not find profile named '{}'".format(profile_name)
|
||
|
|
)
|
||
|
|
|
||
|
|
# First, we've already got our final decision on profile name, and we
|
||
|
|
# don't render keys, so we can pluck that out
|
||
|
|
raw_profile = raw_profiles[profile_name]
|
||
|
|
if not raw_profile:
|
||
|
|
msg = (
|
||
|
|
f'Profile {profile_name} in profiles.yml is empty'
|
||
|
|
)
|
||
|
|
raise DbtProfileError(
|
||
|
|
INVALID_PROFILE_MESSAGE.format(
|
||
|
|
error_string=msg
|
||
|
|
)
|
||
|
|
)
|
||
|
|
user_cfg = raw_profiles.get('config')
|
||
|
|
|
||
|
|
return cls.from_raw_profile_info(
|
||
|
|
raw_profile=raw_profile,
|
||
|
|
profile_name=profile_name,
|
||
|
|
renderer=renderer,
|
||
|
|
user_cfg=user_cfg,
|
||
|
|
target_override=target_override,
|
||
|
|
threads_override=threads_override,
|
||
|
|
)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def render_from_args(
|
||
|
|
cls,
|
||
|
|
args: Any,
|
||
|
|
renderer: ProfileRenderer,
|
||
|
|
project_profile_name: Optional[str],
|
||
|
|
) -> 'Profile':
|
||
|
|
"""Given the raw profiles as read from disk and the name of the desired
|
||
|
|
profile if specified, return the profile component of the runtime
|
||
|
|
config.
|
||
|
|
|
||
|
|
:param args argparse.Namespace: The arguments as parsed from the cli.
|
||
|
|
:param project_profile_name Optional[str]: The profile name, if
|
||
|
|
specified in a project.
|
||
|
|
:raises DbtProjectError: If there is no profile name specified in the
|
||
|
|
project or the command line arguments, or if the specified profile
|
||
|
|
is not found
|
||
|
|
:raises DbtProfileError: If the profile is invalid or missing, or the
|
||
|
|
target could not be found.
|
||
|
|
:returns Profile: The new Profile object.
|
||
|
|
"""
|
||
|
|
threads_override = getattr(args, 'threads', None)
|
||
|
|
target_override = getattr(args, 'target', None)
|
||
|
|
raw_profiles = read_profile(args.profiles_dir)
|
||
|
|
profile_name = cls.pick_profile_name(getattr(args, 'profile', None),
|
||
|
|
project_profile_name)
|
||
|
|
return cls.from_raw_profiles(
|
||
|
|
raw_profiles=raw_profiles,
|
||
|
|
profile_name=profile_name,
|
||
|
|
renderer=renderer,
|
||
|
|
target_override=target_override,
|
||
|
|
threads_override=threads_override
|
||
|
|
)
|