286 lines
9.2 KiB
Python
286 lines
9.2 KiB
Python
|
# special support for CLI argument parsing.
|
||
|
|
from dbt import flags
|
||
|
|
import itertools
|
||
|
|
from dbt.clients.yaml_helper import yaml, Loader, Dumper # noqa: F401
|
||
|
|
|
||
|
|
from typing import (
|
||
|
|
Dict, List, Optional, Tuple, Any, Union
|
||
|
|
)
|
||
|
|
|
||
|
|
from dbt.contracts.selection import SelectorDefinition, SelectorFile
|
||
|
|
from dbt.exceptions import InternalException, ValidationException
|
||
|
|
|
||
|
|
from .selector_spec import (
|
||
|
|
SelectionUnion,
|
||
|
|
SelectionSpec,
|
||
|
|
SelectionIntersection,
|
||
|
|
SelectionDifference,
|
||
|
|
SelectionCriteria,
|
||
|
|
)
|
||
|
|
|
||
|
|
INTERSECTION_DELIMITER = ','
|
||
|
|
|
||
|
|
DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*', 'exposure:*']
|
||
|
|
DEFAULT_EXCLUDES: List[str] = []
|
||
|
|
DATA_TEST_SELECTOR: str = 'test_type:data'
|
||
|
|
SCHEMA_TEST_SELECTOR: str = 'test_type:schema'
|
||
|
|
|
||
|
|
|
||
|
|
def parse_union(
|
||
|
|
components: List[str], expect_exists: bool, greedy: bool = False
|
||
|
|
) -> SelectionUnion:
|
||
|
|
# turn ['a b', 'c'] -> ['a', 'b', 'c']
|
||
|
|
raw_specs = itertools.chain.from_iterable(
|
||
|
|
r.split(' ') for r in components
|
||
|
|
)
|
||
|
|
union_components: List[SelectionSpec] = []
|
||
|
|
|
||
|
|
# ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
|
||
|
|
for raw_spec in raw_specs:
|
||
|
|
intersection_components: List[SelectionSpec] = [
|
||
|
|
SelectionCriteria.from_single_spec(part, greedy=greedy)
|
||
|
|
for part in raw_spec.split(INTERSECTION_DELIMITER)
|
||
|
|
]
|
||
|
|
union_components.append(SelectionIntersection(
|
||
|
|
components=intersection_components,
|
||
|
|
expect_exists=expect_exists,
|
||
|
|
raw=raw_spec,
|
||
|
|
))
|
||
|
|
return SelectionUnion(
|
||
|
|
components=union_components,
|
||
|
|
expect_exists=False,
|
||
|
|
raw=components,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_union_from_default(
|
||
|
|
raw: Optional[List[str]], default: List[str], greedy: bool = False
|
||
|
|
) -> SelectionUnion:
|
||
|
|
components: List[str]
|
||
|
|
expect_exists: bool
|
||
|
|
if raw is None:
|
||
|
|
return parse_union(components=default, expect_exists=False, greedy=greedy)
|
||
|
|
else:
|
||
|
|
return parse_union(components=raw, expect_exists=True, greedy=greedy)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_difference(
|
||
|
|
include: Optional[List[str]], exclude: Optional[List[str]]
|
||
|
|
) -> SelectionDifference:
|
||
|
|
included = parse_union_from_default(include, DEFAULT_INCLUDES, greedy=bool(flags.GREEDY))
|
||
|
|
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, greedy=True)
|
||
|
|
return SelectionDifference(components=[included, excluded])
|
||
|
|
|
||
|
|
|
||
|
|
def parse_test_selectors(
|
||
|
|
data: bool, schema: bool, base: SelectionSpec
|
||
|
|
) -> SelectionSpec:
|
||
|
|
union_components = []
|
||
|
|
|
||
|
|
if data:
|
||
|
|
union_components.append(
|
||
|
|
SelectionCriteria.from_single_spec(DATA_TEST_SELECTOR)
|
||
|
|
)
|
||
|
|
if schema:
|
||
|
|
union_components.append(
|
||
|
|
SelectionCriteria.from_single_spec(SCHEMA_TEST_SELECTOR)
|
||
|
|
)
|
||
|
|
|
||
|
|
intersect_with: SelectionSpec
|
||
|
|
if not union_components:
|
||
|
|
return base
|
||
|
|
elif len(union_components) == 1:
|
||
|
|
intersect_with = union_components[0]
|
||
|
|
else: # data and schema tests
|
||
|
|
intersect_with = SelectionUnion(
|
||
|
|
components=union_components,
|
||
|
|
expect_exists=True,
|
||
|
|
raw=[DATA_TEST_SELECTOR, SCHEMA_TEST_SELECTOR],
|
||
|
|
)
|
||
|
|
|
||
|
|
return SelectionIntersection(
|
||
|
|
components=[base, intersect_with], expect_exists=True
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
RawDefinition = Union[str, Dict[str, Any]]
|
||
|
|
|
||
|
|
|
||
|
|
def _get_list_dicts(
|
||
|
|
dct: Dict[str, Any], key: str
|
||
|
|
) -> List[RawDefinition]:
|
||
|
|
result: List[RawDefinition] = []
|
||
|
|
if key not in dct:
|
||
|
|
raise InternalException(
|
||
|
|
f'Expected to find key {key} in dict, only found {list(dct)}'
|
||
|
|
)
|
||
|
|
values = dct[key]
|
||
|
|
if not isinstance(values, list):
|
||
|
|
raise ValidationException(
|
||
|
|
f'Invalid value for key "{key}". Expected a list.'
|
||
|
|
)
|
||
|
|
for value in values:
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for value_key in value:
|
||
|
|
if not isinstance(value_key, str):
|
||
|
|
raise ValidationException(
|
||
|
|
f'Expected all keys to "{key}" dict to be strings, '
|
||
|
|
f'but "{value_key}" is a "{type(value_key)}"'
|
||
|
|
)
|
||
|
|
result.append(value)
|
||
|
|
elif isinstance(value, str):
|
||
|
|
result.append(value)
|
||
|
|
else:
|
||
|
|
raise ValidationException(
|
||
|
|
f'Invalid value type {type(value)} in key "{key}", expected '
|
||
|
|
f'dict or str (value: {value}).'
|
||
|
|
)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_exclusions(definition) -> Optional[SelectionSpec]:
|
||
|
|
exclusions = _get_list_dicts(definition, 'exclude')
|
||
|
|
parsed_exclusions = [
|
||
|
|
parse_from_definition(excl) for excl in exclusions
|
||
|
|
]
|
||
|
|
if len(parsed_exclusions) == 1:
|
||
|
|
return parsed_exclusions[0]
|
||
|
|
elif len(parsed_exclusions) > 1:
|
||
|
|
return SelectionUnion(
|
||
|
|
components=parsed_exclusions,
|
||
|
|
raw=exclusions
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_include_exclude_subdefs(
|
||
|
|
definitions: List[RawDefinition]
|
||
|
|
) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]:
|
||
|
|
include_parts: List[SelectionSpec] = []
|
||
|
|
diff_arg: Optional[SelectionSpec] = None
|
||
|
|
|
||
|
|
for definition in definitions:
|
||
|
|
if isinstance(definition, dict) and 'exclude' in definition:
|
||
|
|
# do not allow multiple exclude: defs at the same level
|
||
|
|
if diff_arg is not None:
|
||
|
|
yaml_sel_cfg = yaml.dump(definition)
|
||
|
|
raise ValidationException(
|
||
|
|
f"You cannot provide multiple exclude arguments to the "
|
||
|
|
f"same selector set operator:\n{yaml_sel_cfg}"
|
||
|
|
)
|
||
|
|
diff_arg = _parse_exclusions(definition)
|
||
|
|
else:
|
||
|
|
include_parts.append(parse_from_definition(definition))
|
||
|
|
|
||
|
|
return (include_parts, diff_arg)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||
|
|
union_def_parts = _get_list_dicts(definition, 'union')
|
||
|
|
include, exclude = _parse_include_exclude_subdefs(union_def_parts)
|
||
|
|
|
||
|
|
union = SelectionUnion(components=include, greedy_warning=False)
|
||
|
|
|
||
|
|
if exclude is None:
|
||
|
|
union.raw = definition
|
||
|
|
return union
|
||
|
|
else:
|
||
|
|
return SelectionDifference(
|
||
|
|
components=[union, exclude],
|
||
|
|
raw=definition,
|
||
|
|
greedy_warning=False
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_intersection_definition(
|
||
|
|
definition: Dict[str, Any]
|
||
|
|
) -> SelectionSpec:
|
||
|
|
intersection_def_parts = _get_list_dicts(definition, 'intersection')
|
||
|
|
include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
|
||
|
|
intersection = SelectionIntersection(components=include, greedy_warning=False)
|
||
|
|
|
||
|
|
if exclude is None:
|
||
|
|
intersection.raw = definition
|
||
|
|
return intersection
|
||
|
|
else:
|
||
|
|
return SelectionDifference(
|
||
|
|
components=[intersection, exclude],
|
||
|
|
raw=definition,
|
||
|
|
greedy_warning=False
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||
|
|
diff_arg: Optional[SelectionSpec] = None
|
||
|
|
if len(definition) == 1:
|
||
|
|
key = list(definition)[0]
|
||
|
|
value = definition[key]
|
||
|
|
if not isinstance(key, str):
|
||
|
|
raise ValidationException(
|
||
|
|
f'Expected definition key to be a "str", got one of type '
|
||
|
|
f'"{type(key)}" ({key})'
|
||
|
|
)
|
||
|
|
dct = {
|
||
|
|
'method': key,
|
||
|
|
'value': value,
|
||
|
|
}
|
||
|
|
elif 'method' in definition and 'value' in definition:
|
||
|
|
dct = definition
|
||
|
|
if 'exclude' in definition:
|
||
|
|
diff_arg = _parse_exclusions(definition)
|
||
|
|
dct = {k: v for k, v in dct.items() if k != 'exclude'}
|
||
|
|
else:
|
||
|
|
raise ValidationException(
|
||
|
|
f'Expected either 1 key or else "method" '
|
||
|
|
f'and "value" keys, but got {list(definition)}'
|
||
|
|
)
|
||
|
|
|
||
|
|
# if key isn't a valid method name, this will raise
|
||
|
|
base = SelectionCriteria.selection_criteria_from_dict(definition, dct)
|
||
|
|
if diff_arg is None:
|
||
|
|
return base
|
||
|
|
else:
|
||
|
|
return SelectionDifference(components=[base, diff_arg], greedy_warning=False)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_from_definition(
|
||
|
|
definition: RawDefinition, rootlevel=False
|
||
|
|
) -> SelectionSpec:
|
||
|
|
|
||
|
|
if (isinstance(definition, dict) and
|
||
|
|
('union' in definition or 'intersection' in definition) and
|
||
|
|
rootlevel and len(definition) > 1):
|
||
|
|
keys = ",".join(definition.keys())
|
||
|
|
raise ValidationException(
|
||
|
|
f"Only a single 'union' or 'intersection' key is allowed "
|
||
|
|
f"in a root level selector definition; found {keys}."
|
||
|
|
)
|
||
|
|
if isinstance(definition, str):
|
||
|
|
return SelectionCriteria.from_single_spec(definition)
|
||
|
|
elif 'union' in definition:
|
||
|
|
return parse_union_definition(definition)
|
||
|
|
elif 'intersection' in definition:
|
||
|
|
return parse_intersection_definition(definition)
|
||
|
|
elif isinstance(definition, dict):
|
||
|
|
return parse_dict_definition(definition)
|
||
|
|
else:
|
||
|
|
raise ValidationException(
|
||
|
|
f'Expected to find union, intersection, str or dict, instead '
|
||
|
|
f'found {type(definition)}: {definition}'
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_from_selectors_definition(
|
||
|
|
source: SelectorFile
|
||
|
|
) -> Dict[str, Dict[str, Union[SelectionSpec, bool]]]:
|
||
|
|
result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {}
|
||
|
|
selector: SelectorDefinition
|
||
|
|
for selector in source.selectors:
|
||
|
|
result[selector.name] = {
|
||
|
|
"default": selector.default,
|
||
|
|
"definition": parse_from_definition(selector.definition, rootlevel=True)
|
||
|
|
}
|
||
|
|
return result
|