383 lines
11 KiB
Python
383 lines
11 KiB
Python
|
from typing import Dict, Optional, Tuple, Callable
|
||
|
|
from dbt.logger import (
|
||
|
|
GLOBAL_LOGGER as logger,
|
||
|
|
DbtStatusMessage,
|
||
|
|
TextOnly,
|
||
|
|
get_timestamp,
|
||
|
|
)
|
||
|
|
from dbt.node_types import NodeType
|
||
|
|
|
||
|
|
from dbt.tracking import InvocationProcessor
|
||
|
|
from dbt import ui
|
||
|
|
from dbt import utils
|
||
|
|
|
||
|
|
from dbt.contracts.results import (
|
||
|
|
FreshnessStatus, NodeStatus, TestStatus
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def print_fancy_output_line(
|
||
|
|
msg: str, status: str, logger_fn: Callable, index: Optional[int],
|
||
|
|
total: Optional[int], execution_time: Optional[float] = None,
|
||
|
|
truncate: bool = False
|
||
|
|
) -> None:
|
||
|
|
if index is None or total is None:
|
||
|
|
progress = ''
|
||
|
|
else:
|
||
|
|
progress = '{} of {} '.format(index, total)
|
||
|
|
prefix = "{timestamp} | {progress}{message}".format(
|
||
|
|
timestamp=get_timestamp(),
|
||
|
|
progress=progress,
|
||
|
|
message=msg)
|
||
|
|
|
||
|
|
truncate_width = ui.PRINTER_WIDTH - 3
|
||
|
|
justified = prefix.ljust(ui.PRINTER_WIDTH, ".")
|
||
|
|
if truncate and len(justified) > truncate_width:
|
||
|
|
justified = justified[:truncate_width] + '...'
|
||
|
|
|
||
|
|
if execution_time is None:
|
||
|
|
status_time = ""
|
||
|
|
else:
|
||
|
|
status_time = " in {execution_time:0.2f}s".format(
|
||
|
|
execution_time=execution_time)
|
||
|
|
|
||
|
|
output = "{justified} [{status}{status_time}]".format(
|
||
|
|
justified=justified, status=status, status_time=status_time)
|
||
|
|
|
||
|
|
logger_fn(output)
|
||
|
|
|
||
|
|
|
||
|
|
def get_counts(flat_nodes) -> str:
|
||
|
|
counts: Dict[str, int] = {}
|
||
|
|
|
||
|
|
for node in flat_nodes:
|
||
|
|
t = node.resource_type
|
||
|
|
|
||
|
|
if node.resource_type == NodeType.Model:
|
||
|
|
t = '{} {}'.format(node.get_materialization(), t)
|
||
|
|
elif node.resource_type == NodeType.Operation:
|
||
|
|
t = 'hook'
|
||
|
|
|
||
|
|
counts[t] = counts.get(t, 0) + 1
|
||
|
|
|
||
|
|
stat_line = ", ".join(
|
||
|
|
[utils.pluralize(v, k) for k, v in counts.items()])
|
||
|
|
|
||
|
|
return stat_line
|
||
|
|
|
||
|
|
|
||
|
|
def print_start_line(description: str, index: int, total: int) -> None:
|
||
|
|
msg = "START {}".format(description)
|
||
|
|
print_fancy_output_line(msg, 'RUN', logger.info, index, total)
|
||
|
|
|
||
|
|
|
||
|
|
def print_hook_start_line(statement: str, index: int, total: int) -> None:
|
||
|
|
msg = 'START hook: {}'.format(statement)
|
||
|
|
print_fancy_output_line(
|
||
|
|
msg, 'RUN', logger.info, index, total, truncate=True)
|
||
|
|
|
||
|
|
|
||
|
|
def print_hook_end_line(
|
||
|
|
statement: str, status: str, index: int, total: int, execution_time: float
|
||
|
|
) -> None:
|
||
|
|
msg = 'OK hook: {}'.format(statement)
|
||
|
|
# hooks don't fail into this path, so always green
|
||
|
|
print_fancy_output_line(msg, ui.green(status), logger.info, index, total,
|
||
|
|
execution_time=execution_time, truncate=True)
|
||
|
|
|
||
|
|
|
||
|
|
def print_skip_line(
|
||
|
|
node, schema: str, relation: str, index: int, num_models: int
|
||
|
|
) -> None:
|
||
|
|
if node.resource_type in NodeType.refable():
|
||
|
|
msg = f'SKIP relation {schema}.{relation}'
|
||
|
|
else:
|
||
|
|
msg = f'SKIP {node.resource_type} {node.name}'
|
||
|
|
print_fancy_output_line(
|
||
|
|
msg, ui.yellow('SKIP'), logger.info, index, num_models)
|
||
|
|
|
||
|
|
|
||
|
|
def print_cancel_line(model) -> None:
|
||
|
|
msg = 'CANCEL query {}'.format(model)
|
||
|
|
print_fancy_output_line(
|
||
|
|
msg, ui.red('CANCEL'), logger.error, index=None, total=None)
|
||
|
|
|
||
|
|
|
||
|
|
def get_printable_result(
|
||
|
|
result, success: str, error: str) -> Tuple[str, str, Callable]:
|
||
|
|
if result.status == NodeStatus.Error:
|
||
|
|
info = 'ERROR {}'.format(error)
|
||
|
|
status = ui.red(result.status.upper())
|
||
|
|
logger_fn = logger.error
|
||
|
|
else:
|
||
|
|
info = 'OK {}'.format(success)
|
||
|
|
status = ui.green(result.message)
|
||
|
|
logger_fn = logger.info
|
||
|
|
|
||
|
|
return info, status, logger_fn
|
||
|
|
|
||
|
|
|
||
|
|
def print_test_result_line(
|
||
|
|
result, index: int, total: int
|
||
|
|
) -> None:
|
||
|
|
model = result.node
|
||
|
|
|
||
|
|
if result.status == TestStatus.Error:
|
||
|
|
info = "ERROR"
|
||
|
|
color = ui.red
|
||
|
|
logger_fn = logger.error
|
||
|
|
elif result.status == TestStatus.Pass:
|
||
|
|
info = 'PASS'
|
||
|
|
color = ui.green
|
||
|
|
logger_fn = logger.info
|
||
|
|
elif result.status == TestStatus.Warn:
|
||
|
|
info = f'WARN {result.failures}'
|
||
|
|
color = ui.yellow
|
||
|
|
logger_fn = logger.warning
|
||
|
|
elif result.status == TestStatus.Fail:
|
||
|
|
info = f'FAIL {result.failures}'
|
||
|
|
color = ui.red
|
||
|
|
logger_fn = logger.error
|
||
|
|
else:
|
||
|
|
raise RuntimeError("unexpected status: {}".format(result.status))
|
||
|
|
|
||
|
|
print_fancy_output_line(
|
||
|
|
"{info} {name}".format(info=info, name=model.name),
|
||
|
|
color(info),
|
||
|
|
logger_fn,
|
||
|
|
index,
|
||
|
|
total,
|
||
|
|
result.execution_time)
|
||
|
|
|
||
|
|
|
||
|
|
def print_model_result_line(
|
||
|
|
result, description: str, index: int, total: int
|
||
|
|
) -> None:
|
||
|
|
info, status, logger_fn = get_printable_result(
|
||
|
|
result, 'created', 'creating')
|
||
|
|
|
||
|
|
print_fancy_output_line(
|
||
|
|
"{info} {description}".format(info=info, description=description),
|
||
|
|
status,
|
||
|
|
logger_fn,
|
||
|
|
index,
|
||
|
|
total,
|
||
|
|
result.execution_time)
|
||
|
|
|
||
|
|
|
||
|
|
def print_snapshot_result_line(
|
||
|
|
result, description: str, index: int, total: int
|
||
|
|
) -> None:
|
||
|
|
model = result.node
|
||
|
|
|
||
|
|
info, status, logger_fn = get_printable_result(
|
||
|
|
result, 'snapshotted', 'snapshotting')
|
||
|
|
cfg = model.config.to_dict(omit_none=True)
|
||
|
|
|
||
|
|
msg = "{info} {description}".format(
|
||
|
|
info=info, description=description, **cfg)
|
||
|
|
print_fancy_output_line(
|
||
|
|
msg,
|
||
|
|
status,
|
||
|
|
logger_fn,
|
||
|
|
index,
|
||
|
|
total,
|
||
|
|
result.execution_time)
|
||
|
|
|
||
|
|
|
||
|
|
def print_seed_result_line(result, schema_name: str, index: int, total: int):
|
||
|
|
model = result.node
|
||
|
|
|
||
|
|
info, status, logger_fn = get_printable_result(result, 'loaded', 'loading')
|
||
|
|
|
||
|
|
print_fancy_output_line(
|
||
|
|
"{info} seed file {schema}.{relation}".format(
|
||
|
|
info=info,
|
||
|
|
schema=schema_name,
|
||
|
|
relation=model.alias),
|
||
|
|
status,
|
||
|
|
logger_fn,
|
||
|
|
index,
|
||
|
|
total,
|
||
|
|
result.execution_time)
|
||
|
|
|
||
|
|
|
||
|
|
def print_freshness_result_line(result, index: int, total: int) -> None:
|
||
|
|
if result.status == FreshnessStatus.RuntimeErr:
|
||
|
|
info = 'ERROR'
|
||
|
|
color = ui.red
|
||
|
|
logger_fn = logger.error
|
||
|
|
elif result.status == FreshnessStatus.Error:
|
||
|
|
info = 'ERROR STALE'
|
||
|
|
color = ui.red
|
||
|
|
logger_fn = logger.error
|
||
|
|
elif result.status == FreshnessStatus.Warn:
|
||
|
|
info = 'WARN'
|
||
|
|
color = ui.yellow
|
||
|
|
logger_fn = logger.warning
|
||
|
|
else:
|
||
|
|
info = 'PASS'
|
||
|
|
color = ui.green
|
||
|
|
logger_fn = logger.info
|
||
|
|
|
||
|
|
if hasattr(result, 'node'):
|
||
|
|
source_name = result.node.source_name
|
||
|
|
table_name = result.node.name
|
||
|
|
else:
|
||
|
|
source_name = result.source_name
|
||
|
|
table_name = result.table_name
|
||
|
|
|
||
|
|
msg = f"{info} freshness of {source_name}.{table_name}"
|
||
|
|
|
||
|
|
print_fancy_output_line(
|
||
|
|
msg,
|
||
|
|
color(info),
|
||
|
|
logger_fn,
|
||
|
|
index,
|
||
|
|
total,
|
||
|
|
execution_time=result.execution_time
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def interpret_run_result(result) -> str:
|
||
|
|
if result.status in (NodeStatus.Error, NodeStatus.Fail):
|
||
|
|
return 'error'
|
||
|
|
elif result.status == NodeStatus.Skipped:
|
||
|
|
return 'skip'
|
||
|
|
elif result.status == NodeStatus.Warn:
|
||
|
|
return 'warn'
|
||
|
|
elif result.status in (NodeStatus.Pass, NodeStatus.Success):
|
||
|
|
return 'pass'
|
||
|
|
else:
|
||
|
|
raise RuntimeError(f"unhandled result {result}")
|
||
|
|
|
||
|
|
|
||
|
|
def print_run_status_line(results) -> None:
|
||
|
|
stats = {
|
||
|
|
'error': 0,
|
||
|
|
'skip': 0,
|
||
|
|
'pass': 0,
|
||
|
|
'warn': 0,
|
||
|
|
'total': 0,
|
||
|
|
}
|
||
|
|
|
||
|
|
for r in results:
|
||
|
|
result_type = interpret_run_result(r)
|
||
|
|
stats[result_type] += 1
|
||
|
|
stats['total'] += 1
|
||
|
|
|
||
|
|
stats_line = "\nDone. PASS={pass} WARN={warn} ERROR={error} SKIP={skip} TOTAL={total}" # noqa
|
||
|
|
logger.info(stats_line.format(**stats))
|
||
|
|
|
||
|
|
|
||
|
|
def print_run_result_error(
|
||
|
|
result, newline: bool = True, is_warning: bool = False
|
||
|
|
) -> None:
|
||
|
|
if newline:
|
||
|
|
with TextOnly():
|
||
|
|
logger.info("")
|
||
|
|
|
||
|
|
if result.status == NodeStatus.Fail or (
|
||
|
|
is_warning and result.status == NodeStatus.Warn
|
||
|
|
):
|
||
|
|
if is_warning:
|
||
|
|
color = ui.yellow
|
||
|
|
info = 'Warning'
|
||
|
|
logger_fn = logger.warning
|
||
|
|
else:
|
||
|
|
color = ui.red
|
||
|
|
info = 'Failure'
|
||
|
|
logger_fn = logger.error
|
||
|
|
logger_fn(color("{} in {} {} ({})").format(
|
||
|
|
info,
|
||
|
|
result.node.resource_type,
|
||
|
|
result.node.name,
|
||
|
|
result.node.original_file_path))
|
||
|
|
|
||
|
|
if result.message:
|
||
|
|
logger.error(f" {result.message}")
|
||
|
|
else:
|
||
|
|
logger.error(f" Status: {result.status}")
|
||
|
|
|
||
|
|
if result.node.build_path is not None:
|
||
|
|
with TextOnly():
|
||
|
|
logger.info("")
|
||
|
|
logger.info(" compiled SQL at {}".format(
|
||
|
|
result.node.compiled_path))
|
||
|
|
|
||
|
|
if result.node.should_store_failures:
|
||
|
|
with TextOnly():
|
||
|
|
logger.info("")
|
||
|
|
msg = f"select * from {result.node.relation_name}"
|
||
|
|
border = '-' * len(msg)
|
||
|
|
logger.info(f" See test failures:\n {border}\n {msg}\n {border}")
|
||
|
|
|
||
|
|
elif result.message is not None:
|
||
|
|
first = True
|
||
|
|
for line in result.message.split("\n"):
|
||
|
|
if first:
|
||
|
|
logger.error(ui.yellow(line))
|
||
|
|
first = False
|
||
|
|
else:
|
||
|
|
logger.error(line)
|
||
|
|
|
||
|
|
|
||
|
|
def print_skip_caused_by_error(
|
||
|
|
model, schema: str, relation: str, index: int, num_models: int, result
|
||
|
|
) -> None:
|
||
|
|
msg = ('SKIP relation {}.{} due to ephemeral model error'
|
||
|
|
.format(schema, relation))
|
||
|
|
print_fancy_output_line(
|
||
|
|
msg, ui.red('ERROR SKIP'), logger.error, index, num_models)
|
||
|
|
print_run_result_error(result, newline=False)
|
||
|
|
|
||
|
|
|
||
|
|
def print_end_of_run_summary(
|
||
|
|
num_errors: int, num_warnings: int, keyboard_interrupt: bool = False
|
||
|
|
) -> None:
|
||
|
|
error_plural = utils.pluralize(num_errors, 'error')
|
||
|
|
warn_plural = utils.pluralize(num_warnings, 'warning')
|
||
|
|
if keyboard_interrupt:
|
||
|
|
message = ui.yellow('Exited because of keyboard interrupt.')
|
||
|
|
elif num_errors > 0:
|
||
|
|
message = ui.red("Completed with {} and {}:".format(
|
||
|
|
error_plural, warn_plural))
|
||
|
|
elif num_warnings > 0:
|
||
|
|
message = ui.yellow('Completed with {}:'.format(warn_plural))
|
||
|
|
else:
|
||
|
|
message = ui.green('Completed successfully')
|
||
|
|
|
||
|
|
with TextOnly():
|
||
|
|
logger.info('')
|
||
|
|
logger.info('{}'.format(message))
|
||
|
|
|
||
|
|
|
||
|
|
def print_run_end_messages(results, keyboard_interrupt: bool = False) -> None:
|
||
|
|
errors, warnings = [], []
|
||
|
|
for r in results:
|
||
|
|
if r.status in (
|
||
|
|
NodeStatus.RuntimeErr,
|
||
|
|
NodeStatus.Error,
|
||
|
|
NodeStatus.Fail
|
||
|
|
):
|
||
|
|
errors.append(r)
|
||
|
|
elif r.status == NodeStatus.Skipped and r.message is not None:
|
||
|
|
# this means we skipped a node because of an issue upstream,
|
||
|
|
# so include it as an error
|
||
|
|
errors.append(r)
|
||
|
|
elif r.status == NodeStatus.Warn:
|
||
|
|
warnings.append(r)
|
||
|
|
|
||
|
|
with DbtStatusMessage(), InvocationProcessor():
|
||
|
|
print_end_of_run_summary(len(errors),
|
||
|
|
len(warnings),
|
||
|
|
keyboard_interrupt)
|
||
|
|
|
||
|
|
for error in errors:
|
||
|
|
print_run_result_error(error, is_warning=False)
|
||
|
|
|
||
|
|
for warning in warnings:
|
||
|
|
print_run_result_error(warning, is_warning=True)
|
||
|
|
|
||
|
|
print_run_status_line(results)
|