172 lines
5.5 KiB
Python
172 lines
5.5 KiB
Python
import os
|
|
import hashlib
|
|
from typing import List, Optional
|
|
|
|
from dbt.clients import git, system
|
|
from dbt.config import Project
|
|
from dbt.contracts.project import (
|
|
ProjectPackageMetadata,
|
|
GitPackage,
|
|
)
|
|
from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
|
|
from dbt.exceptions import (
|
|
ExecutableError, warn_or_error, raise_dependency_error
|
|
)
|
|
from dbt.logger import GLOBAL_LOGGER as logger
|
|
from dbt import ui
|
|
|
|
PIN_PACKAGE_URL = 'https://docs.getdbt.com/docs/package-management#section-specifying-package-versions' # noqa
|
|
|
|
|
|
def md5sum(s: str):
|
|
return hashlib.md5(s.encode('latin-1')).hexdigest()
|
|
|
|
|
|
class GitPackageMixin:
|
|
def __init__(self, git: str) -> None:
|
|
super().__init__()
|
|
self.git = git
|
|
|
|
@property
|
|
def name(self):
|
|
return self.git
|
|
|
|
def source_type(self) -> str:
|
|
return 'git'
|
|
|
|
|
|
class GitPinnedPackage(GitPackageMixin, PinnedPackage):
|
|
def __init__(
|
|
self,
|
|
git: str,
|
|
revision: str,
|
|
warn_unpinned: bool = True,
|
|
subdirectory: Optional[str] = None,
|
|
) -> None:
|
|
super().__init__(git)
|
|
self.revision = revision
|
|
self.warn_unpinned = warn_unpinned
|
|
self.subdirectory = subdirectory
|
|
self._checkout_name = md5sum(self.git)
|
|
|
|
def get_version(self):
|
|
return self.revision
|
|
|
|
def get_subdirectory(self):
|
|
return self.subdirectory
|
|
|
|
def nice_version_name(self):
|
|
if self.revision == 'HEAD':
|
|
return 'HEAD (default revision)'
|
|
else:
|
|
return 'revision {}'.format(self.revision)
|
|
|
|
def unpinned_msg(self):
|
|
if self.revision == 'HEAD':
|
|
return 'not pinned, using HEAD (default branch)'
|
|
elif self.revision in ('main', 'master'):
|
|
return f'pinned to the "{self.revision}" branch'
|
|
else:
|
|
return None
|
|
|
|
def _checkout(self):
|
|
"""Performs a shallow clone of the repository into the downloads
|
|
directory. This function can be called repeatedly. If the project has
|
|
already been checked out at this version, it will be a no-op. Returns
|
|
the path to the checked out directory."""
|
|
try:
|
|
dir_ = git.clone_and_checkout(
|
|
self.git, get_downloads_path(), revision=self.revision,
|
|
dirname=self._checkout_name, subdirectory=self.subdirectory
|
|
)
|
|
except ExecutableError as exc:
|
|
if exc.cmd and exc.cmd[0] == 'git':
|
|
logger.error(
|
|
'Make sure git is installed on your machine. More '
|
|
'information: '
|
|
'https://docs.getdbt.com/docs/package-management'
|
|
)
|
|
raise
|
|
return os.path.join(get_downloads_path(), dir_)
|
|
|
|
def _fetch_metadata(self, project, renderer) -> ProjectPackageMetadata:
|
|
path = self._checkout()
|
|
|
|
if self.unpinned_msg() and self.warn_unpinned:
|
|
warn_or_error(
|
|
'The git package "{}" \n\tis {}.\n\tThis can introduce '
|
|
'breaking changes into your project without warning!\n\nSee {}'
|
|
.format(self.git, self.unpinned_msg(), PIN_PACKAGE_URL),
|
|
log_fmt=ui.yellow('WARNING: {}')
|
|
)
|
|
loaded = Project.from_project_root(path, renderer)
|
|
return ProjectPackageMetadata.from_project(loaded)
|
|
|
|
def install(self, project, renderer):
|
|
dest_path = self.get_installation_path(project, renderer)
|
|
if os.path.exists(dest_path):
|
|
if system.path_is_symlink(dest_path):
|
|
system.remove_file(dest_path)
|
|
else:
|
|
system.rmdir(dest_path)
|
|
|
|
system.move(self._checkout(), dest_path)
|
|
|
|
|
|
class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
|
|
def __init__(
|
|
self,
|
|
git: str,
|
|
revisions: List[str],
|
|
warn_unpinned: bool = True,
|
|
subdirectory: Optional[str] = None,
|
|
) -> None:
|
|
super().__init__(git)
|
|
self.revisions = revisions
|
|
self.warn_unpinned = warn_unpinned
|
|
self.subdirectory = subdirectory
|
|
|
|
@classmethod
|
|
def from_contract(
|
|
cls, contract: GitPackage
|
|
) -> 'GitUnpinnedPackage':
|
|
revisions = contract.get_revisions()
|
|
|
|
# we want to map None -> True
|
|
warn_unpinned = contract.warn_unpinned is not False
|
|
return cls(git=contract.git, revisions=revisions,
|
|
warn_unpinned=warn_unpinned, subdirectory=contract.subdirectory)
|
|
|
|
def all_names(self) -> List[str]:
|
|
if self.git.endswith('.git'):
|
|
other = self.git[:-4]
|
|
else:
|
|
other = self.git + '.git'
|
|
return [self.git, other]
|
|
|
|
def incorporate(
|
|
self, other: 'GitUnpinnedPackage'
|
|
) -> 'GitUnpinnedPackage':
|
|
warn_unpinned = self.warn_unpinned and other.warn_unpinned
|
|
|
|
return GitUnpinnedPackage(
|
|
git=self.git,
|
|
revisions=self.revisions + other.revisions,
|
|
warn_unpinned=warn_unpinned,
|
|
subdirectory=self.subdirectory,
|
|
)
|
|
|
|
def resolved(self) -> GitPinnedPackage:
|
|
requested = set(self.revisions)
|
|
if len(requested) == 0:
|
|
requested = {'HEAD'}
|
|
elif len(requested) > 1:
|
|
raise_dependency_error(
|
|
'git dependencies should contain exactly one version. '
|
|
'{} contains: {}'.format(self.git, requested))
|
|
|
|
return GitPinnedPackage(
|
|
git=self.git, revision=requested.pop(),
|
|
warn_unpinned=self.warn_unpinned, subdirectory=self.subdirectory
|
|
)
|