from typing import MutableMapping, Dict, List
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.files import (
    AnySourceFile, ParseFileType, parse_file_type_to_parser,
)
from dbt.logger import GLOBAL_LOGGER as logger
from dbt.node_types import NodeType


mssat_files = (
    ParseFileType.Model,
    ParseFileType.Seed,
    ParseFileType.Snapshot,
    ParseFileType.Analysis,
    ParseFileType.Test,
)


key_to_prefix = {
    'models': 'model',
    'seeds': 'seed',
    'snapshots': 'snapshot',
    'analyses': 'analysis',
}


parse_file_type_to_key = {
    ParseFileType.Model: 'models',
    ParseFileType.Seed: 'seeds',
    ParseFileType.Snapshot: 'snapshots',
    ParseFileType.Analysis: 'analyses',
}


# Partial parsing. Create a diff of files from saved manifest and current
# files and produce a project_parser_file dictionary to drive parsing of
# only the necessary changes.
# Will produce a 'skip_parsing' method, and a project_parser_file dictionary
class PartialParsing:
    def __init__(self, saved_manifest: Manifest, new_files: MutableMapping[str, AnySourceFile]):
        self.saved_manifest = saved_manifest
        self.new_files = new_files
        self.project_parser_files: Dict = {}
        self.saved_files = self.saved_manifest.files
        self.project_parser_files = {}
        self.deleted_manifest = Manifest()
        self.macro_child_map: Dict[str, List[str]] = {}
        self.build_file_diff()
        self.processing_file = None
        self.disabled_by_file_id = self.saved_manifest.build_disabled_by_file_id()

    def skip_parsing(self):
        return (
            not self.file_diff['deleted'] and
            not self.file_diff['added'] and
            not self.file_diff['changed'] and
            not self.file_diff['changed_schema_files'] and
            not self.file_diff['deleted_schema_files']
        )

    # Compare the previously saved manifest files and the just-loaded manifest
    # files to see if anything changed
    def build_file_diff(self):
        saved_file_ids = set(self.saved_files.keys())
        new_file_ids = set(self.new_files.keys())
        deleted_all_files = saved_file_ids.difference(new_file_ids)
        added = new_file_ids.difference(saved_file_ids)
        common = saved_file_ids.intersection(new_file_ids)
        changed_or_deleted_macro_file = False

        # separate out deleted schema files
        deleted_schema_files = []
        deleted = []
        for file_id in deleted_all_files:
            if self.saved_files[file_id].parse_file_type == ParseFileType.Schema:
                deleted_schema_files.append(file_id)
            else:
                if self.saved_files[file_id].parse_file_type == ParseFileType.Macro:
                    changed_or_deleted_macro_file = True
                deleted.append(file_id)

        changed = []
        changed_schema_files = []
        unchanged = []
        for file_id in common:
            if self.saved_files[file_id].checksum == self.new_files[file_id].checksum:
                unchanged.append(file_id)
            else:
                # separate out changed schema files
                if self.saved_files[file_id].parse_file_type == ParseFileType.Schema:
                    sf = self.saved_files[file_id]
                    if type(sf).__name__ != 'SchemaSourceFile':
                        raise Exception(f"Serialization failure for {file_id}")
                    changed_schema_files.append(file_id)
                else:
                    if self.saved_files[file_id].parse_file_type == ParseFileType.Macro:
                        changed_or_deleted_macro_file = True
                    changed.append(file_id)
        file_diff = {
            "deleted": deleted,
            "deleted_schema_files": deleted_schema_files,
            "added": added,
            "changed": changed,
            "changed_schema_files": changed_schema_files,
            "unchanged": unchanged,
        }
        if changed_or_deleted_macro_file:
            self.macro_child_map = self.saved_manifest.build_macro_child_map()
        logger.info(f"Partial parsing enabled: "
                    f"{len(deleted) + len(deleted_schema_files)} files deleted, "
                    f"{len(added)} files added, "
                    f"{len(changed) + len(changed_schema_files)} files changed.")
        self.file_diff = file_diff

    # generate the list of files that need parsing
    # uses self.manifest.files generated by 'read_files'
    def get_parsing_files(self):
        if self.skip_parsing():
            return {}
        # Need to add new files first, because changes in schema files
        # might refer to them
        for file_id in self.file_diff['added']:
            self.processing_file = file_id
            self.add_to_saved(file_id)
        # Need to process schema files next, because the dictionaries
        # need to be in place for handling SQL file changes
        for file_id in self.file_diff['changed_schema_files']:
            self.processing_file = file_id
            self.change_schema_file(file_id)
        for file_id in self.file_diff['deleted_schema_files']:
            self.processing_file = file_id
            self.delete_schema_file(file_id)
        for file_id in self.file_diff['deleted']:
            self.processing_file = file_id
            self.delete_from_saved(file_id)
        for file_id in self.file_diff['changed']:
            self.processing_file = file_id
            self.update_in_saved(file_id)
        return self.project_parser_files

    # Add the file to the project parser dictionaries to schedule parsing
    def add_to_pp_files(self, source_file):
        file_id = source_file.file_id
        parser_name = parse_file_type_to_parser[source_file.parse_file_type]
        project_name = source_file.project_name
        if not parser_name or not project_name:
            raise Exception(f"Did not find parse_file_type or project_name "
                            f"in SourceFile for {source_file.file_id}")
        if project_name not in self.project_parser_files:
            self.project_parser_files[project_name] = {}
        if parser_name not in self.project_parser_files[project_name]:
            self.project_parser_files[project_name][parser_name] = []
        if (file_id not in self.project_parser_files[project_name][parser_name] and
                file_id not in self.file_diff['deleted']):
            self.project_parser_files[project_name][parser_name].append(file_id)

    def already_scheduled_for_parsing(self, source_file):
        file_id = source_file.file_id
        project_name = source_file.project_name
        if project_name not in self.project_parser_files:
            return False
        parser_name = parse_file_type_to_parser[source_file.parse_file_type]
        if parser_name not in self.project_parser_files[project_name]:
            return False
        if file_id not in self.project_parser_files[project_name][parser_name]:
            return False
        return True

    # Add new files, including schema files
    def add_to_saved(self, file_id):
        # add file object to saved manifest.files
        source_file = self.new_files[file_id]
        if source_file.parse_file_type == ParseFileType.Schema:
            self.handle_added_schema_file(source_file)
        self.saved_files[file_id] = source_file
        # update pp_files to parse
        self.add_to_pp_files(source_file)
        logger.debug(f"Partial parsing: added file: {file_id}")

    def handle_added_schema_file(self, source_file):
        source_file.pp_dict = source_file.dict_from_yaml.copy()
        if 'sources' in source_file.pp_dict:
            for source in source_file.pp_dict['sources']:
                # We need to remove the original source, so it can
                # be properly patched
                if 'overrides' in source:
                    self.remove_source_override_target(source)

    # Deletes for all non-schema files
    def delete_from_saved(self, file_id):
        # Look at all things touched by file, remove those
        # nodes, and update pp_files to parse unless the
        # file creating those nodes has also been deleted
        saved_source_file = self.saved_files[file_id]

        # SQL file: models, seeds, snapshots, analyses, tests: SQL files, except
        # macros/tests
        if saved_source_file.parse_file_type in mssat_files:
            self.remove_mssat_file(saved_source_file)
            self.deleted_manifest.files[file_id] = self.saved_manifest.files.pop(file_id)

        # macros
        if saved_source_file.parse_file_type == ParseFileType.Macro:
            self.delete_macro_file(saved_source_file, follow_references=True)

        # docs
        if saved_source_file.parse_file_type == ParseFileType.Documentation:
            self.delete_doc_node(saved_source_file)

        logger.debug(f"Partial parsing: deleted file: {file_id}")

    # Updates for non-schema files
    def update_in_saved(self, file_id):
        new_source_file = self.new_files[file_id]
        old_source_file = self.saved_files[file_id]

        if new_source_file.parse_file_type in mssat_files:
            self.update_mssat_in_saved(new_source_file, old_source_file)
        elif new_source_file.parse_file_type == ParseFileType.Macro:
            self.update_macro_in_saved(new_source_file, old_source_file)
        elif new_source_file.parse_file_type == ParseFileType.Documentation:
            self.update_doc_in_saved(new_source_file, old_source_file)
        else:
            raise Exception(f"Invalid parse_file_type in source_file {file_id}")
        logger.debug(f"Partial parsing: updated file: {file_id}")

    # Models, seeds, snapshots: patches and tests
    # analyses: patches, no tests
    # tests: not touched by schema files (no patches, no tests)
    # Updated schema files should have been processed already.
    def update_mssat_in_saved(self, new_source_file, old_source_file):

        if self.already_scheduled_for_parsing(old_source_file):
            return

        # These files only have one node.
        unique_id = None
        if old_source_file.nodes:
            unique_id = old_source_file.nodes[0]
        else:
            # It's not clear when this would actually happen.
            # Logging in case there are other associated errors.
            logger.debug(f"Partial parsing: node not found for source_file {old_source_file}")

        # replace source_file in saved and add to parsing list
        file_id = new_source_file.file_id
        self.deleted_manifest.files[file_id] = old_source_file
        self.saved_files[file_id] = new_source_file
        self.add_to_pp_files(new_source_file)
        if unique_id:
            self.remove_node_in_saved(new_source_file, unique_id)

    def remove_node_in_saved(self, source_file, unique_id):
        if unique_id in self.saved_manifest.nodes:
            # delete node in saved
            node = self.saved_manifest.nodes.pop(unique_id)
            self.deleted_manifest.nodes[unique_id] = node
        elif source_file.file_id in self.disabled_by_file_id:
            for dis_index, dis_node in self.saved_manifest.disabled:
                if dis_node.file_id == source_file.file_id:
                    node = dis_node
                    break
            if dis_node:
                del self.saved_manifest.disabled[dis_index]
        else:
            # Has already been deleted by another action
            return

        # look at patch_path in model node to see if we need
        # to reapply a patch from a schema_file.
        if node.patch_path:
            file_id = node.patch_path
            # it might be changed...  then what?
            if file_id not in self.file_diff['deleted']:
                # schema_files should already be updated
                schema_file = self.saved_files[file_id]
                dict_key = parse_file_type_to_key[source_file.parse_file_type]
                # look for a matching list dictionary
                elem_patch = None
                if dict_key in schema_file.dict_from_yaml:
                    for elem in schema_file.dict_from_yaml[dict_key]:
                        if elem['name'] == node.name:
                            elem_patch = elem
                            break
                if elem_patch:
                    self.delete_schema_mssa_links(schema_file, dict_key, elem_patch)
                    self.merge_patch(schema_file, dict_key, elem_patch)
                    if unique_id in schema_file.node_patches:
                        schema_file.node_patches.remove(unique_id)
            if unique_id in self.saved_manifest.disabled:
                # We have a patch_path in disabled nodes with a patch so
                # that we can connect the patch to the node
                for node in self.saved_manifest.disabled[unique_id]:
                    node.patch_path = None

    def update_macro_in_saved(self, new_source_file, old_source_file):
        if self.already_scheduled_for_parsing(old_source_file):
            return
        self.handle_macro_file_links(old_source_file, follow_references=True)
        file_id = new_source_file.file_id
        self.saved_files[file_id] = new_source_file
        self.add_to_pp_files(new_source_file)

    def update_doc_in_saved(self, new_source_file, old_source_file):
        if self.already_scheduled_for_parsing(old_source_file):
            return
        self.delete_doc_node(old_source_file)
        self.saved_files[new_source_file.file_id] = new_source_file
        self.add_to_pp_files(new_source_file)

    def remove_mssat_file(self, source_file):
        # nodes [unique_ids] -- SQL files
        # There should always be a node for a SQL file
        if not source_file.nodes:
            logger.debug(f"No nodes found for source file {source_file.file_id}")
            return
        # There is generally only 1 node for SQL files, except for macros
        for unique_id in source_file.nodes:
            self.remove_node_in_saved(source_file, unique_id)
            self.schedule_referencing_nodes_for_parsing(unique_id)

    # We need to re-parse nodes that reference another removed node
    def schedule_referencing_nodes_for_parsing(self, unique_id):
        # Look at "children", i.e. nodes that reference this node
        if unique_id in self.saved_manifest.child_map:
            self.schedule_nodes_for_parsing(self.saved_manifest.child_map[unique_id])
        else:
            logger.debug(f"Partial parsing: {unique_id} not found in child_map")

    def schedule_nodes_for_parsing(self, unique_ids):
        for unique_id in unique_ids:
            if unique_id in self.saved_manifest.nodes:
                node = self.saved_manifest.nodes[unique_id]
                if node.resource_type == NodeType.Test:
                    # test nodes are handled separately. Must be removed from schema file
                    continue
                file_id = node.file_id
                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                    source_file = self.saved_files[file_id]
                    self.remove_mssat_file(source_file)
                    # content of non-schema files is only in new files
                    self.saved_files[file_id] = self.new_files[file_id]
                    self.add_to_pp_files(self.saved_files[file_id])
            elif unique_id in self.saved_manifest.sources:
                source = self.saved_manifest.sources[unique_id]
                file_id = source.file_id
                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                    schema_file = self.saved_files[file_id]
                    sources = []
                    if 'sources' in schema_file.dict_from_yaml:
                        sources = schema_file.dict_from_yaml['sources']
                    source_element = self.get_schema_element(sources, source.source_name)
                    if source_element:
                        self.delete_schema_source(schema_file, source_element)
                        self.remove_tests(schema_file, 'sources', source_element['name'])
                        self.merge_patch(schema_file, 'sources', source_element)
            elif unique_id in self.saved_manifest.exposures:
                exposure = self.saved_manifest.exposures[unique_id]
                file_id = exposure.file_id
                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                    schema_file = self.saved_files[file_id]
                    exposures = []
                    if 'exposures' in schema_file.dict_from_yaml:
                        exposures = schema_file.dict_from_yaml['exposures']
                    exposure_element = self.get_schema_element(exposures, exposure.name)
                    if exposure_element:
                        self.delete_schema_exposure(schema_file, exposure_element)
                        self.merge_patch(schema_file, 'exposures', exposure_element)
            elif unique_id in self.saved_manifest.macros:
                macro = self.saved_manifest.macros[unique_id]
                file_id = macro.file_id
                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                    source_file = self.saved_files[file_id]
                    self.delete_macro_file(source_file)
                    self.saved_files[file_id] = self.new_files[file_id]
                    self.add_to_pp_files(self.saved_files[file_id])

    def delete_macro_file(self, source_file, follow_references=False):
        self.handle_macro_file_links(source_file, follow_references)
        file_id = source_file.file_id
        self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)

    def recursively_gather_macro_references(self, macro_unique_id, referencing_nodes):
        for unique_id in self.macro_child_map[macro_unique_id]:
            if unique_id in referencing_nodes:
                continue
            referencing_nodes.append(unique_id)
            if unique_id.startswith('macro.'):
                self.recursively_gather_macro_references(unique_id, referencing_nodes)

    def handle_macro_file_links(self, source_file, follow_references=False):
        # remove the macros in the 'macros' dictionary
        macros = source_file.macros.copy()
        for unique_id in macros:
            if unique_id not in self.saved_manifest.macros:
                # This happens when a macro has already been removed
                if unique_id in source_file.macros:
                    source_file.macros.remove(unique_id)
                continue

            base_macro = self.saved_manifest.macros.pop(unique_id)
            self.deleted_manifest.macros[unique_id] = base_macro

            # Recursively check children of this macro
            # The macro_child_map might not exist if a macro is removed by
            # schedule_nodes_for parsing. We only want to follow
            # references if the macro file itself has been updated or
            # deleted, not if we're just updating referenced nodes.
            if self.macro_child_map and follow_references:
                referencing_nodes = []
                self.recursively_gather_macro_references(unique_id, referencing_nodes)
                self.schedule_macro_nodes_for_parsing(referencing_nodes)

            if base_macro.patch_path:
                file_id = base_macro.patch_path
                if file_id in self.saved_files:
                    schema_file = self.saved_files[file_id]
                    macro_patches = []
                    if 'macros' in schema_file.dict_from_yaml:
                        macro_patches = schema_file.dict_from_yaml['macros']
                    macro_patch = self.get_schema_element(macro_patches, base_macro.name)
                    self.delete_schema_macro_patch(schema_file, macro_patch)
                    self.merge_patch(schema_file, 'macros', macro_patch)
            # The macro may have already been removed by handling macro children
            if unique_id in source_file.macros:
                source_file.macros.remove(unique_id)

    # similar to schedule_nodes_for_parsing but doesn't do sources and exposures
    # and handles schema tests
    def schedule_macro_nodes_for_parsing(self, unique_ids):
        for unique_id in unique_ids:
            if unique_id in self.saved_manifest.nodes:
                node = self.saved_manifest.nodes[unique_id]
                if node.resource_type == NodeType.Test:
                    schema_file_id = node.file_id
                    schema_file = self.saved_manifest.files[schema_file_id]
                    (key, name) = schema_file.get_key_and_name_for_test(node.unique_id)
                    if key and name:
                        patch_list = []
                        if key in schema_file.dict_from_yaml:
                            patch_list = schema_file.dict_from_yaml[key]
                        patch = self.get_schema_element(patch_list, name)
                        if patch:
                            if key in ['models', 'seeds', 'snapshots']:
                                self.delete_schema_mssa_links(schema_file, key, patch)
                                self.merge_patch(schema_file, key, patch)
                                if unique_id in schema_file.node_patches:
                                    schema_file.node_patches.remove(unique_id)
                            elif key == 'sources':
                                # re-schedule source
                                if 'overrides' in patch:
                                    # This is a source patch; need to re-parse orig source
                                    self.remove_source_override_target(patch)
                                self.delete_schema_source(schema_file, patch)
                                self.remove_tests(schema_file, 'sources', patch['name'])
                                self.merge_patch(schema_file, 'sources', patch)
                else:
                    file_id = node.file_id
                    if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                        source_file = self.saved_files[file_id]
                        self.remove_mssat_file(source_file)
                        # content of non-schema files is only in new files
                        self.saved_files[file_id] = self.new_files[file_id]
                        self.add_to_pp_files(self.saved_files[file_id])
            elif unique_id in self.saved_manifest.macros:
                macro = self.saved_manifest.macros[unique_id]
                file_id = macro.file_id
                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
                    source_file = self.saved_files[file_id]
                    self.delete_macro_file(source_file)
                    self.saved_files[file_id] = self.new_files[file_id]
                    self.add_to_pp_files(self.saved_files[file_id])

    def delete_doc_node(self, source_file):
        # remove the nodes in the 'docs' dictionary
        docs = source_file.docs.copy()
        for unique_id in docs:
            self.deleted_manifest.docs[unique_id] = self.saved_manifest.docs.pop(unique_id)
            source_file.docs.remove(unique_id)
        # The unique_id of objects that contain a doc call are stored in the
        # doc source_file.nodes
        self.schedule_nodes_for_parsing(source_file.nodes)
        source_file.nodes = []

    # Schema files -----------------------
    # Changed schema files
    def change_schema_file(self, file_id):
        saved_schema_file = self.saved_files[file_id]
        new_schema_file = self.new_files[file_id]
        saved_yaml_dict = saved_schema_file.dict_from_yaml
        new_yaml_dict = new_schema_file.dict_from_yaml
        if 'version' in new_yaml_dict:
            # despite the fact that this goes in the saved_schema_file, it
            # should represent the new yaml dictionary, and should produce
            # an error if the updated yaml file doesn't have a version
            saved_schema_file.pp_dict = {"version": new_yaml_dict['version']}
        else:
            saved_schema_file.pp_dict = {}
        self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)

        # copy from new schema_file to saved_schema_file to preserve references
        # that weren't removed
        saved_schema_file.contents = new_schema_file.contents
        saved_schema_file.checksum = new_schema_file.checksum
        saved_schema_file.dfy = new_schema_file.dfy
        # schedule parsing
        self.add_to_pp_files(saved_schema_file)
        # schema_file pp_dict should have been generated already
        logger.debug(f"Partial parsing: update schema file: {file_id}")

    # Delete schema files -- a variation on change_schema_file
    def delete_schema_file(self, file_id):
        saved_schema_file = self.saved_files[file_id]
        saved_yaml_dict = saved_schema_file.dict_from_yaml
        new_yaml_dict = {}
        self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)
        self.deleted_manifest.files[file_id] = self.saved_manifest.files.pop(file_id)

    # For each key in a schema file dictionary, process the changed, deleted, and added
    # elemnts for the key lists
    def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict):
        # loop through comparing previous dict_from_yaml with current dict_from_yaml
        # Need to do the deleted/added/changed thing, just like the files lists

        # models, seeds, snapshots, analyses
        for dict_key in ['models', 'seeds', 'snapshots', 'analyses']:
            key_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict)
            if key_diff['changed']:
                for elem in key_diff['changed']:
                    self.delete_schema_mssa_links(schema_file, dict_key, elem)
                    self.merge_patch(schema_file, dict_key, elem)
            if key_diff['deleted']:
                for elem in key_diff['deleted']:
                    self.delete_schema_mssa_links(schema_file, dict_key, elem)
            if key_diff['added']:
                for elem in key_diff['added']:
                    self.merge_patch(schema_file, dict_key, elem)

        # sources
        source_diff = self.get_diff_for('sources', saved_yaml_dict, new_yaml_dict)
        if source_diff['changed']:
            for source in source_diff['changed']:
                if 'overrides' in source:  # This is a source patch; need to re-parse orig source
                    self.remove_source_override_target(source)
                self.delete_schema_source(schema_file, source)
                self.remove_tests(schema_file, 'sources', source['name'])
                self.merge_patch(schema_file, 'sources', source)
        if source_diff['deleted']:
            for source in source_diff['deleted']:
                if 'overrides' in source:  # This is a source patch; need to re-parse orig source
                    self.remove_source_override_target(source)
                self.delete_schema_source(schema_file, source)
                self.remove_tests(schema_file, 'sources', source['name'])
        if source_diff['added']:
            for source in source_diff['added']:
                if 'overrides' in source:  # This is a source patch; need to re-parse orig source
                    self.remove_source_override_target(source)
                self.merge_patch(schema_file, 'sources', source)

        # macros
        macro_diff = self.get_diff_for('macros', saved_yaml_dict, new_yaml_dict)
        if macro_diff['changed']:
            for macro in macro_diff['changed']:
                self.delete_schema_macro_patch(schema_file, macro)
                self.merge_patch(schema_file, 'macros', macro)
        if macro_diff['deleted']:
            for macro in macro_diff['deleted']:
                self.delete_schema_macro_patch(schema_file, macro)
        if macro_diff['added']:
            for macro in macro_diff['added']:
                self.merge_patch(schema_file, 'macros', macro)

        # exposures
        exposure_diff = self.get_diff_for('exposures', saved_yaml_dict, new_yaml_dict)
        if exposure_diff['changed']:
            for exposure in exposure_diff['changed']:
                self.delete_schema_exposure(schema_file, exposure)
                self.merge_patch(schema_file, 'exposures', exposure)
        if exposure_diff['deleted']:
            for exposure in exposure_diff['deleted']:
                self.delete_schema_exposure(schema_file, exposure)
        if exposure_diff['added']:
            for exposure in exposure_diff['added']:
                self.merge_patch(schema_file, 'exposures', exposure)

    # Take a "section" of the schema file yaml dictionary from saved and new schema files
    # and determine which parts have changed
    def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict):
        if key in saved_yaml_dict or key in new_yaml_dict:
            saved_elements = saved_yaml_dict[key] if key in saved_yaml_dict else []
            new_elements = new_yaml_dict[key] if key in new_yaml_dict else []
        else:
            return {'deleted': [], 'added': [], 'changed': []}
        # for each set of keys, need to create a dictionary of names pointing to entry
        saved_elements_by_name = {}
        new_elements_by_name = {}
        # sources have two part names?
        for element in saved_elements:
            saved_elements_by_name[element['name']] = element
        for element in new_elements:
            new_elements_by_name[element['name']] = element

        # now determine which elements, by name, are added, deleted or changed
        saved_element_names = set(saved_elements_by_name.keys())
        new_element_names = set(new_elements_by_name.keys())
        deleted = saved_element_names.difference(new_element_names)
        added = new_element_names.difference(saved_element_names)
        common = saved_element_names.intersection(new_element_names)
        changed = []
        for element_name in common:
            if saved_elements_by_name[element_name] != new_elements_by_name[element_name]:
                changed.append(element_name)

        # make lists of yaml elements to return as diffs
        deleted_elements = [saved_elements_by_name[name].copy() for name in deleted]
        added_elements = [new_elements_by_name[name].copy() for name in added]
        changed_elements = [new_elements_by_name[name].copy() for name in changed]

        diff = {
            "deleted": deleted_elements,
            "added": added_elements,
            "changed": changed_elements,
        }
        return diff

    # Merge a patch file into the pp_dict in a schema file
    def merge_patch(self, schema_file, key, patch):
        if not schema_file.pp_dict:
            schema_file.pp_dict = {"version": schema_file.dict_from_yaml['version']}
        pp_dict = schema_file.pp_dict
        if key not in pp_dict:
            pp_dict[key] = [patch]
        else:
            # check that this patch hasn't already been saved
            found = False
            for elem in pp_dict[key]:
                if elem['name'] == patch['name']:
                    found = True
            if not found:
                pp_dict[key].append(patch)
        self.add_to_pp_files(schema_file)

    # For model, seed, snapshot, analysis schema dictionary keys,
    # delete the patches and tests from the patch
    def delete_schema_mssa_links(self, schema_file, dict_key, elem):
        # find elem node unique_id in node_patches
        prefix = key_to_prefix[dict_key]
        elem_unique_id = ''
        for unique_id in schema_file.node_patches:
            if not unique_id.startswith(prefix):
                continue
            parts = unique_id.split('.')
            elem_name = parts[-1]
            if elem_name == elem['name']:
                elem_unique_id = unique_id
                break

        # remove elem node and remove unique_id from node_patches
        if elem_unique_id:
            # might have been already removed
            if elem_unique_id in self.saved_manifest.nodes:
                node = self.saved_manifest.nodes.pop(elem_unique_id)
                self.deleted_manifest.nodes[elem_unique_id] = node
                # need to add the node source_file to pp_files
                file_id = node.file_id
                # need to copy new file to saved files in order to get content
                if file_id in self.new_files:
                    self.saved_files[file_id] = self.new_files[file_id]
                if self.saved_files[file_id]:
                    source_file = self.saved_files[file_id]
                    self.add_to_pp_files(source_file)
            # remove from patches
            schema_file.node_patches.remove(elem_unique_id)

        # for models, seeds, snapshots (not analyses)
        if dict_key in ['models', 'seeds', 'snapshots']:
            # find related tests and remove them
            self.remove_tests(schema_file, dict_key, elem['name'])

    def remove_tests(self, schema_file, dict_key, name):
        tests = schema_file.get_tests(dict_key, name)
        for test_unique_id in tests:
            if test_unique_id in self.saved_manifest.nodes:
                node = self.saved_manifest.nodes.pop(test_unique_id)
                self.deleted_manifest.nodes[test_unique_id] = node
        schema_file.remove_tests(dict_key, name)

    def delete_schema_source(self, schema_file, source_dict):
        # both patches, tests, and source nodes
        source_name = source_dict['name']
        # There may be multiple sources for each source dict, since
        # there will be a separate source node for each table.
        # ParsedSourceDefinition name = table name, dict name is source_name
        sources = schema_file.sources.copy()
        for unique_id in sources:
            if unique_id in self.saved_manifest.sources:
                source = self.saved_manifest.sources[unique_id]
                if source.source_name == source_name:
                    source = self.saved_manifest.sources.pop(unique_id)
                    self.deleted_manifest.sources[unique_id] = source
                    schema_file.sources.remove(unique_id)
                    self.schedule_referencing_nodes_for_parsing(unique_id)
                    logger.debug(f"Partial parsing: deleted source {unique_id}")

    def delete_schema_macro_patch(self, schema_file, macro):
        # This is just macro patches that need to be reapplied
        macro_unique_id = None
        if macro['name'] in schema_file.macro_patches:
            macro_unique_id = schema_file.macro_patches[macro['name']]
            del schema_file.macro_patches[macro['name']]
        if macro_unique_id and macro_unique_id in self.saved_manifest.macros:
            macro = self.saved_manifest.macros.pop(macro_unique_id)
            self.deleted_manifest.macros[macro_unique_id] = macro
            macro_file_id = macro.file_id
            if macro_file_id in self.new_files:
                self.saved_files[macro_file_id] = self.new_files[macro_file_id]
                self.add_to_pp_files(self.saved_files[macro_file_id])

    # exposures are created only from schema files, so just delete
    # the exposure.
    def delete_schema_exposure(self, schema_file, exposure_dict):
        exposure_name = exposure_dict['name']
        exposures = schema_file.exposures.copy()
        for unique_id in exposures:
            exposure = self.saved_manifest.exposures[unique_id]
            if unique_id in self.saved_manifest.exposures:
                if exposure.name == exposure_name:
                    self.deleted_manifest.exposures[unique_id] = \
                        self.saved_manifest.exposures.pop(unique_id)
                    schema_file.exposures.remove(unique_id)
                    logger.debug(f"Partial parsing: deleted exposure {unique_id}")

    def get_schema_element(self, elem_list, elem_name):
        for element in elem_list:
            if 'name' in element and element['name'] == elem_name:
                return element
        return None

    def get_schema_file_for_source(self, package_name, source_name):
        schema_file = None
        for source in self.saved_manifest.sources.values():
            if source.package_name == package_name and source.source_name == source_name:
                file_id = source.file_id
                if file_id in self.saved_files:
                    schema_file = self.saved_files[file_id]
                break
        return schema_file

    def get_source_override_file_and_dict(self, source):
        package = source['overrides']
        source_name = source['name']
        orig_source_schema_file = self.get_schema_file_for_source(package, source_name)
        orig_sources = orig_source_schema_file.dict_from_yaml['sources']
        orig_source = self.get_schema_element(orig_sources, source_name)
        return (orig_source_schema_file, orig_source)

    def remove_source_override_target(self, source_dict):
        (orig_file, orig_source) = self.get_source_override_file_and_dict(source_dict)
        if orig_source:
            self.delete_schema_source(orig_file, orig_source)
            self.remove_tests(orig_file, 'sources', orig_source['name'])
            self.merge_patch(orig_file, 'sources', orig_source)
            self.add_to_pp_files(orig_file)
