410 lines
14 KiB
Python
410 lines
14 KiB
Python
|
# coding=utf-8
|
||
|
|
import os
|
||
|
|
import platform
|
||
|
|
import sys
|
||
|
|
from typing import Optional, Dict, Any, List
|
||
|
|
|
||
|
|
from dbt.logger import GLOBAL_LOGGER as logger
|
||
|
|
import dbt.clients.system
|
||
|
|
import dbt.exceptions
|
||
|
|
from dbt.adapters.factory import get_adapter, register_adapter
|
||
|
|
from dbt.config import Project, Profile, PROFILES_DIR
|
||
|
|
from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
|
||
|
|
from dbt.config.utils import parse_cli_vars
|
||
|
|
from dbt.context.base import generate_base_context
|
||
|
|
from dbt.context.target import generate_target_context
|
||
|
|
from dbt.clients.yaml_helper import load_yaml_text
|
||
|
|
from dbt.links import ProfileConfigDocs
|
||
|
|
from dbt.ui import green, red
|
||
|
|
from dbt.utils import pluralize
|
||
|
|
from dbt.version import get_installed_version
|
||
|
|
|
||
|
|
from dbt.task.base import BaseTask, get_nearest_project_dir
|
||
|
|
|
||
|
|
PROFILE_DIR_MESSAGE = """To view your profiles.yml file, run:
|
||
|
|
|
||
|
|
{open_cmd} {profiles_dir}"""
|
||
|
|
|
||
|
|
ONLY_PROFILE_MESSAGE = '''
|
||
|
|
A `dbt_project.yml` file was not found in this directory.
|
||
|
|
Using the only profile `{}`.
|
||
|
|
'''.lstrip()
|
||
|
|
|
||
|
|
MULTIPLE_PROFILE_MESSAGE = '''
|
||
|
|
A `dbt_project.yml` file was not found in this directory.
|
||
|
|
dbt found the following profiles:
|
||
|
|
{}
|
||
|
|
|
||
|
|
To debug one of these profiles, run:
|
||
|
|
dbt debug --profile [profile-name]
|
||
|
|
'''.lstrip()
|
||
|
|
|
||
|
|
COULD_NOT_CONNECT_MESSAGE = '''
|
||
|
|
dbt was unable to connect to the specified database.
|
||
|
|
The database returned the following error:
|
||
|
|
|
||
|
|
>{err}
|
||
|
|
|
||
|
|
Check your database credentials and try again. For more information, visit:
|
||
|
|
{url}
|
||
|
|
'''.lstrip()
|
||
|
|
|
||
|
|
MISSING_PROFILE_MESSAGE = '''
|
||
|
|
dbt looked for a profiles.yml file in {path}, but did
|
||
|
|
not find one. For more information on configuring your profile, consult the
|
||
|
|
documentation:
|
||
|
|
|
||
|
|
{url}
|
||
|
|
'''.lstrip()
|
||
|
|
|
||
|
|
FILE_NOT_FOUND = 'file not found'
|
||
|
|
|
||
|
|
|
||
|
|
class QueryCommentedProfile(Profile):
|
||
|
|
def __init__(self, *args, **kwargs):
|
||
|
|
super().__init__(*args, **kwargs)
|
||
|
|
self.query_comment = None
|
||
|
|
|
||
|
|
|
||
|
|
class DebugTask(BaseTask):
|
||
|
|
def __init__(self, args, config):
|
||
|
|
super().__init__(args, config)
|
||
|
|
self.profiles_dir = getattr(self.args, 'profiles_dir', PROFILES_DIR)
|
||
|
|
self.profile_path = os.path.join(self.profiles_dir, 'profiles.yml')
|
||
|
|
try:
|
||
|
|
self.project_dir = get_nearest_project_dir(self.args)
|
||
|
|
except dbt.exceptions.Exception:
|
||
|
|
# we probably couldn't find a project directory. Set project dir
|
||
|
|
# to whatever was given, or default to the current directory.
|
||
|
|
if args.project_dir:
|
||
|
|
self.project_dir = args.project_dir
|
||
|
|
else:
|
||
|
|
self.project_dir = os.getcwd()
|
||
|
|
self.project_path = os.path.join(self.project_dir, 'dbt_project.yml')
|
||
|
|
self.cli_vars = parse_cli_vars(getattr(self.args, 'vars', '{}'))
|
||
|
|
|
||
|
|
# set by _load_*
|
||
|
|
self.profile: Optional[Profile] = None
|
||
|
|
self.profile_fail_details = ''
|
||
|
|
self.raw_profile_data: Optional[Dict[str, Any]] = None
|
||
|
|
self.profile_name: Optional[str] = None
|
||
|
|
self.project: Optional[Project] = None
|
||
|
|
self.project_fail_details = ''
|
||
|
|
self.any_failure = False
|
||
|
|
self.messages: List[str] = []
|
||
|
|
|
||
|
|
@property
|
||
|
|
def project_profile(self):
|
||
|
|
if self.project is None:
|
||
|
|
return None
|
||
|
|
return self.project.profile_name
|
||
|
|
|
||
|
|
def path_info(self):
|
||
|
|
open_cmd = dbt.clients.system.open_dir_cmd()
|
||
|
|
|
||
|
|
message = PROFILE_DIR_MESSAGE.format(
|
||
|
|
open_cmd=open_cmd,
|
||
|
|
profiles_dir=self.profiles_dir
|
||
|
|
)
|
||
|
|
|
||
|
|
logger.info(message)
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
if self.args.config_dir:
|
||
|
|
self.path_info()
|
||
|
|
return not self.any_failure
|
||
|
|
|
||
|
|
version = get_installed_version().to_version_string(skip_matcher=True)
|
||
|
|
print('dbt version: {}'.format(version))
|
||
|
|
print('python version: {}'.format(sys.version.split()[0]))
|
||
|
|
print('python path: {}'.format(sys.executable))
|
||
|
|
print('os info: {}'.format(platform.platform()))
|
||
|
|
print('Using profiles.yml file at {}'.format(self.profile_path))
|
||
|
|
print('Using dbt_project.yml file at {}'.format(self.project_path))
|
||
|
|
print('')
|
||
|
|
self.test_configuration()
|
||
|
|
self.test_dependencies()
|
||
|
|
self.test_connection()
|
||
|
|
|
||
|
|
if self.any_failure:
|
||
|
|
print(red(f"{(pluralize(len(self.messages), 'check'))} failed:"))
|
||
|
|
else:
|
||
|
|
print(green('All checks passed!'))
|
||
|
|
|
||
|
|
for message in self.messages:
|
||
|
|
print(message)
|
||
|
|
print('')
|
||
|
|
|
||
|
|
return not self.any_failure
|
||
|
|
|
||
|
|
def interpret_results(self, results):
|
||
|
|
return results
|
||
|
|
|
||
|
|
def _load_project(self):
|
||
|
|
if not os.path.exists(self.project_path):
|
||
|
|
self.project_fail_details = FILE_NOT_FOUND
|
||
|
|
return red('ERROR not found')
|
||
|
|
|
||
|
|
if self.profile is None:
|
||
|
|
ctx = generate_base_context(self.cli_vars)
|
||
|
|
else:
|
||
|
|
ctx = generate_target_context(self.profile, self.cli_vars)
|
||
|
|
|
||
|
|
renderer = DbtProjectYamlRenderer(ctx)
|
||
|
|
|
||
|
|
try:
|
||
|
|
self.project = Project.from_project_root(
|
||
|
|
self.project_dir,
|
||
|
|
renderer,
|
||
|
|
verify_version=getattr(self.args, 'version_check', False),
|
||
|
|
)
|
||
|
|
except dbt.exceptions.DbtConfigError as exc:
|
||
|
|
self.project_fail_details = str(exc)
|
||
|
|
return red('ERROR invalid')
|
||
|
|
|
||
|
|
return green('OK found and valid')
|
||
|
|
|
||
|
|
def _profile_found(self):
|
||
|
|
if not self.raw_profile_data:
|
||
|
|
return red('ERROR not found')
|
||
|
|
assert self.raw_profile_data is not None
|
||
|
|
if self.profile_name in self.raw_profile_data:
|
||
|
|
return green('OK found')
|
||
|
|
else:
|
||
|
|
return red('ERROR not found')
|
||
|
|
|
||
|
|
def _target_found(self):
|
||
|
|
requirements = (self.raw_profile_data and self.profile_name and
|
||
|
|
self.target_name)
|
||
|
|
if not requirements:
|
||
|
|
return red('ERROR not found')
|
||
|
|
# mypy appeasement, we checked just above
|
||
|
|
assert self.raw_profile_data is not None
|
||
|
|
assert self.profile_name is not None
|
||
|
|
assert self.target_name is not None
|
||
|
|
if self.profile_name not in self.raw_profile_data:
|
||
|
|
return red('ERROR not found')
|
||
|
|
profiles = self.raw_profile_data[self.profile_name]['outputs']
|
||
|
|
if self.target_name not in profiles:
|
||
|
|
return red('ERROR not found')
|
||
|
|
return green('OK found')
|
||
|
|
|
||
|
|
def _choose_profile_names(self) -> Optional[List[str]]:
|
||
|
|
project_profile: Optional[str] = None
|
||
|
|
if os.path.exists(self.project_path):
|
||
|
|
try:
|
||
|
|
partial = Project.partial_load(
|
||
|
|
os.path.dirname(self.project_path),
|
||
|
|
verify_version=getattr(self.args, 'version_check', False),
|
||
|
|
)
|
||
|
|
renderer = DbtProjectYamlRenderer(
|
||
|
|
generate_base_context(self.cli_vars)
|
||
|
|
)
|
||
|
|
project_profile = partial.render_profile_name(renderer)
|
||
|
|
except dbt.exceptions.DbtProjectError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
args_profile: Optional[str] = getattr(self.args, 'profile', None)
|
||
|
|
|
||
|
|
try:
|
||
|
|
return [Profile.pick_profile_name(args_profile, project_profile)]
|
||
|
|
except dbt.exceptions.DbtConfigError:
|
||
|
|
pass
|
||
|
|
# try to guess
|
||
|
|
|
||
|
|
profiles = []
|
||
|
|
if self.raw_profile_data:
|
||
|
|
profiles = [k for k in self.raw_profile_data if k != 'config']
|
||
|
|
if project_profile is None:
|
||
|
|
self.messages.append('Could not load dbt_project.yml')
|
||
|
|
elif len(profiles) == 0:
|
||
|
|
self.messages.append('The profiles.yml has no profiles')
|
||
|
|
elif len(profiles) == 1:
|
||
|
|
self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
|
||
|
|
else:
|
||
|
|
self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
|
||
|
|
'\n'.join(' - {}'.format(o) for o in profiles)
|
||
|
|
))
|
||
|
|
return profiles
|
||
|
|
|
||
|
|
def _choose_target_name(self, profile_name: str):
|
||
|
|
has_raw_profile = (
|
||
|
|
self.raw_profile_data is not None and
|
||
|
|
profile_name in self.raw_profile_data
|
||
|
|
)
|
||
|
|
|
||
|
|
if not has_raw_profile:
|
||
|
|
return None
|
||
|
|
|
||
|
|
# mypy appeasement, we checked just above
|
||
|
|
assert self.raw_profile_data is not None
|
||
|
|
raw_profile = self.raw_profile_data[profile_name]
|
||
|
|
|
||
|
|
renderer = ProfileRenderer(generate_base_context(self.cli_vars))
|
||
|
|
|
||
|
|
target_name, _ = Profile.render_profile(
|
||
|
|
raw_profile=raw_profile,
|
||
|
|
profile_name=profile_name,
|
||
|
|
target_override=getattr(self.args, 'target', None),
|
||
|
|
renderer=renderer
|
||
|
|
)
|
||
|
|
return target_name
|
||
|
|
|
||
|
|
def _load_profile(self):
|
||
|
|
if not os.path.exists(self.profile_path):
|
||
|
|
self.profile_fail_details = FILE_NOT_FOUND
|
||
|
|
self.messages.append(MISSING_PROFILE_MESSAGE.format(
|
||
|
|
path=self.profile_path, url=ProfileConfigDocs
|
||
|
|
))
|
||
|
|
self.any_failure = True
|
||
|
|
return red('ERROR not found')
|
||
|
|
|
||
|
|
try:
|
||
|
|
raw_profile_data = load_yaml_text(
|
||
|
|
dbt.clients.system.load_file_contents(self.profile_path)
|
||
|
|
)
|
||
|
|
except Exception:
|
||
|
|
pass # we'll report this when we try to load the profile for real
|
||
|
|
else:
|
||
|
|
if isinstance(raw_profile_data, dict):
|
||
|
|
self.raw_profile_data = raw_profile_data
|
||
|
|
|
||
|
|
profile_errors = []
|
||
|
|
profile_names = self._choose_profile_names()
|
||
|
|
renderer = ProfileRenderer(generate_base_context(self.cli_vars))
|
||
|
|
for profile_name in profile_names:
|
||
|
|
try:
|
||
|
|
profile: Profile = QueryCommentedProfile.render_from_args(
|
||
|
|
self.args, renderer, profile_name
|
||
|
|
)
|
||
|
|
except dbt.exceptions.DbtConfigError as exc:
|
||
|
|
profile_errors.append(str(exc))
|
||
|
|
else:
|
||
|
|
if len(profile_names) == 1:
|
||
|
|
# if a profile was specified, set it on the task
|
||
|
|
self.target_name = self._choose_target_name(profile_name)
|
||
|
|
self.profile = profile
|
||
|
|
|
||
|
|
if profile_errors:
|
||
|
|
self.profile_fail_details = '\n\n'.join(profile_errors)
|
||
|
|
return red('ERROR invalid')
|
||
|
|
return green('OK found and valid')
|
||
|
|
|
||
|
|
def test_git(self):
|
||
|
|
try:
|
||
|
|
dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help'])
|
||
|
|
except dbt.exceptions.ExecutableError as exc:
|
||
|
|
self.messages.append('Error from git --help: {!s}'.format(exc))
|
||
|
|
self.any_failure = True
|
||
|
|
return red('ERROR')
|
||
|
|
return green('OK found')
|
||
|
|
|
||
|
|
def test_dependencies(self):
|
||
|
|
print('Required dependencies:')
|
||
|
|
print(' - git [{}]'.format(self.test_git()))
|
||
|
|
print('')
|
||
|
|
|
||
|
|
def test_configuration(self):
|
||
|
|
profile_status = self._load_profile()
|
||
|
|
project_status = self._load_project()
|
||
|
|
print('Configuration:')
|
||
|
|
print(' profiles.yml file [{}]'.format(profile_status))
|
||
|
|
print(' dbt_project.yml file [{}]'.format(project_status))
|
||
|
|
# skip profile stuff if we can't find a profile name
|
||
|
|
if self.profile_name is not None:
|
||
|
|
print(' profile: {} [{}]'.format(self.profile_name,
|
||
|
|
self._profile_found()))
|
||
|
|
print(' target: {} [{}]'.format(self.target_name,
|
||
|
|
self._target_found()))
|
||
|
|
print('')
|
||
|
|
self._log_project_fail()
|
||
|
|
self._log_profile_fail()
|
||
|
|
|
||
|
|
def _log_project_fail(self):
|
||
|
|
if not self.project_fail_details:
|
||
|
|
return
|
||
|
|
|
||
|
|
self.any_failure = True
|
||
|
|
if self.project_fail_details == FILE_NOT_FOUND:
|
||
|
|
return
|
||
|
|
msg = (
|
||
|
|
f'Project loading failed for the following reason:'
|
||
|
|
f'\n{self.project_fail_details}'
|
||
|
|
f'\n'
|
||
|
|
)
|
||
|
|
self.messages.append(msg)
|
||
|
|
|
||
|
|
def _log_profile_fail(self):
|
||
|
|
if not self.profile_fail_details:
|
||
|
|
return
|
||
|
|
|
||
|
|
self.any_failure = True
|
||
|
|
if self.profile_fail_details == FILE_NOT_FOUND:
|
||
|
|
return
|
||
|
|
msg = (
|
||
|
|
f'Profile loading failed for the following reason:'
|
||
|
|
f'\n{self.profile_fail_details}'
|
||
|
|
f'\n'
|
||
|
|
)
|
||
|
|
self.messages.append(msg)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def attempt_connection(profile):
|
||
|
|
"""Return a string containing the error message, or None if there was
|
||
|
|
no error.
|
||
|
|
"""
|
||
|
|
register_adapter(profile)
|
||
|
|
adapter = get_adapter(profile)
|
||
|
|
try:
|
||
|
|
with adapter.connection_named('debug'):
|
||
|
|
adapter.debug_query()
|
||
|
|
except Exception as exc:
|
||
|
|
return COULD_NOT_CONNECT_MESSAGE.format(
|
||
|
|
err=str(exc),
|
||
|
|
url=ProfileConfigDocs,
|
||
|
|
)
|
||
|
|
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _connection_result(self):
|
||
|
|
result = self.attempt_connection(self.profile)
|
||
|
|
if result is not None:
|
||
|
|
self.messages.append(result)
|
||
|
|
self.any_failure = True
|
||
|
|
return red('ERROR')
|
||
|
|
return green('OK connection ok')
|
||
|
|
|
||
|
|
def test_connection(self):
|
||
|
|
if not self.profile:
|
||
|
|
return
|
||
|
|
print('Connection:')
|
||
|
|
for k, v in self.profile.credentials.connection_info():
|
||
|
|
print(' {}: {}'.format(k, v))
|
||
|
|
print(' Connection test: [{}]'.format(self._connection_result()))
|
||
|
|
print('')
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def validate_connection(cls, target_dict):
|
||
|
|
"""Validate a connection dictionary. On error, raises a DbtConfigError.
|
||
|
|
"""
|
||
|
|
target_name = 'test'
|
||
|
|
# make a fake profile that we can parse
|
||
|
|
profile_data = {
|
||
|
|
'outputs': {
|
||
|
|
target_name: target_dict,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
# this will raise a DbtConfigError on failure
|
||
|
|
profile = Profile.from_raw_profile_info(
|
||
|
|
raw_profile=profile_data,
|
||
|
|
profile_name='',
|
||
|
|
target_override=target_name,
|
||
|
|
renderer=ProfileRenderer(generate_base_context({})),
|
||
|
|
)
|
||
|
|
result = cls.attempt_connection(profile)
|
||
|
|
if result is not None:
|
||
|
|
raise dbt.exceptions.DbtProfileError(
|
||
|
|
result,
|
||
|
|
result_type='connection_failure'
|
||
|
|
)
|