OVMS3-idf/tools/ldgen/generation.py

#
# Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import re
import collections
import itertools
import os
import subprocess
import fnmatch

from sdkconfig import SDKConfig
from fragments import FragmentFileModel, Sections, Scheme, Mapping, Fragment
from pyparsing import *
from common import LdGenFailure

"""
Encapsulates a generated placement rule placed under a target
"""
class PlacementRule():

    DEFAULT_SPECIFICITY = 0
    ARCHIVE_SPECIFICITY = 1
    OBJECT_SPECIFICITY = 2
    SYMBOL_SPECIFICITY = 3

    class __container():
        def __init__(self, content):
            self.content = content

    __metadata = collections.namedtuple("__metadata", "excludes expansions expanded")

    def __init__(self, archive, obj, symbol, sections, target):
        if archive == "*":
            archive = None

        if obj == "*":
            obj = None

        self.archive = archive
        self.obj = obj
        self.symbol = symbol
        self.target = target
        self.sections = dict()

        self.specificity = 0
        self.specificity += 1 if self.archive else 0
        self.specificity += 1 if (self.obj and not self.obj == '*') else 0
        self.specificity += 1 if self.symbol else 0

        for section in sections:
            section_data = Sections.get_section_data_from_entry(section, self.symbol)

            if not self.symbol:
                for s in section_data:
                    metadata = self.__metadata(self.__container([]), self.__container([]), self.__container(False))
                    self.sections[s] = metadata
            else:
                (section, expansion) = section_data
                if expansion:
                    metadata = self.__metadata(self.__container([]), self.__container([expansion]), self.__container(True))
                    self.sections[section] =  metadata

    def get_section_names(self):
        return self.sections.keys()

    def add_exclusion(self, other, sections_infos = None):
        # Utility functions for this method
        def do_section_expansion(rule, section):
            if section in rule.get_section_names():
                sections_in_obj = sections_infos.get_obj_sections(rule.archive, rule.obj)

                expansions = fnmatch.filter(sections_in_obj, section)
                return expansions

        def remove_section_expansions(rule, section, expansions):
            existing_expansions = self.sections[section].expansions.content
            self.sections[section].expansions.content = [e for e in existing_expansions if e not in expansions]

        # Exit immediately if the exclusion to be added is more general than this rule.
        if not other.is_more_specific_rule_of(self):
            return

        for section in self.get_sections_intersection(other):
            if(other.specificity == PlacementRule.SYMBOL_SPECIFICITY):
                # If this sections has not been expanded previously, expand now and keep track.
                previously_expanded = self.sections[section].expanded.content
                if not previously_expanded:
                    expansions = do_section_expansion(self, section)
                    if expansions:
                        self.sections[section].expansions.content = expansions
                        self.sections[section].expanded.content = True
                        previously_expanded = True

                # Remove the sections corresponding to the symbol name
                remove_section_expansions(self, section, other.sections[section].expansions.content)

                # If it has been expanded previously but now the expansions list is empty,
                # it means adding exclusions has exhausted the list. Remove the section entirely.
                if previously_expanded and not self.sections[section].expanded.content:
                    del self.sections[section]
            else:
                # A rule section can have multiple rule sections excluded from it. Get the
                # most specific rule from the list, and if an even more specific rule is found,
                # replace it entirely. Otherwise, keep appending.
                exclusions = self.sections[section].excludes
                exclusions_list = exclusions.content if exclusions.content != None else []
                exclusions_to_remove = filter(lambda r: r.is_more_specific_rule_of(other), exclusions_list)

                remaining_exclusions = [e for e in exclusions_list if e not in exclusions_to_remove]
                remaining_exclusions.append(other)

                self.sections[section].excludes.content = remaining_exclusions

    def get_sections_intersection(self, other):
        return set(self.sections.keys()).intersection(set(other.sections.keys()))

    def is_more_specific_rule_of(self, other):
        if (self.specificity <= other.specificity):
            return False

        # Compare archive, obj and target
        for entity_index in range (1, other.specificity + 1):
            if self[entity_index] != other[entity_index] and other[entity_index] != None:
                return False

        return True

    def maps_same_entities_as(self, other):
        if self.specificity != other.specificity:
            return False

        # Compare archive, obj and target
        for entity_index in range (1, other.specificity + 1):
            if self[entity_index] != other[entity_index] and other[entity_index] != None:
                return False

        return True

    def __getitem__(self, key):
        if key == PlacementRule.ARCHIVE_SPECIFICITY:
            return  self.archive
        elif key == PlacementRule.OBJECT_SPECIFICITY:
            return self.obj
        elif key == PlacementRule.SYMBOL_SPECIFICITY:
            return self.symbol
        else:
            return None

    def __str__(self):
        sorted_sections = sorted(self.get_section_names())

        sections_string = list()

        for section in sorted_sections:
            exclusions = self.sections[section].excludes.content

            exclusion_string = None

            if exclusions:
                exclusion_string = " ".join(map(lambda e: "*" + e.archive + (":" + e.obj + ".*" if e.obj else ""), exclusions))
                exclusion_string = "EXCLUDE_FILE(" + exclusion_string + ")"
            else:
                exclusion_string = ""

            section_string = None
            exclusion_section_string = None

            section_expansions = self.sections[section].expansions.content
            section_expanded = self.sections[section].expanded.content

            if section_expansions and section_expanded:
                section_string = " ".join(section_expansions)
                exclusion_section_string = section_string
            else:
                section_string = section
                exclusion_section_string = exclusion_string + " " + section_string

            sections_string.append(exclusion_section_string)

        sections_string = " ".join(sections_string)

        archive = str(self.archive) if self.archive else ""
        obj = (str(self.obj) +  (".*" if self.obj else "")) if self.obj else ""

        # Handle output string generation based on information available
        if self.specificity == PlacementRule.DEFAULT_SPECIFICITY:
            rule_string = "*(%s)" % (sections_string)
        elif self.specificity == PlacementRule.ARCHIVE_SPECIFICITY:
            rule_string = "*%s:(%s)" % (archive, sections_string)
        else:
            rule_string = "*%s:%s(%s)" % (archive, obj, sections_string)

        return rule_string

    def __eq__(self, other):
        if id(self) == id(other):
            return True

        def exclusions_set(exclusions):
            exclusions_set = {(e.archive, e.obj, e.symbol, e.target) for e in exclusions}
            return exclusions_set

        if self.archive != other.archive:
            return False

        if self.obj != other.obj:
            return False

        if self.symbol != other.symbol:
            return False

        if set(self.sections.keys()) != set(other.sections.keys()):
            return False

        for (section, metadata) in self.sections.items():

            self_meta = metadata
            other_meta = other.sections[section]

            if exclusions_set(self_meta.excludes.content) != exclusions_set(other_meta.excludes.content):
                return False

            if set(self_meta.expansions.content) != set(other_meta.expansions.content):
                return False

        return True

    def __ne__(self, other):
        return not self.__eq__(other)

    def __iter__(self):
        yield self.archive
        yield self.obj
        yield self.symbol
        raise StopIteration

"""
Implements generation of placement rules based on collected sections, scheme and mapping fragment.
"""
class GenerationModel:

    DEFAULT_SCHEME = "default"

    def __init__(self):
        self.schemes = {}
        self.sections = {}
        self.mappings = {}

    def _add_mapping_rules(self, archive, obj, symbol, scheme_name, scheme_dict, rules):
        # Use an ordinary dictionary to raise exception on non-existing keys
        temp_dict = dict(scheme_dict)

        sections_bucket = temp_dict[scheme_name]

        for (target, sections) in sections_bucket.items():
            section_entries = []

            for section in sections:
                section_entries.extend(section.entries)

            rule = PlacementRule(archive, obj, symbol, section_entries, target)

            if not rule in rules:
                rules.append(rule)

    def _build_scheme_dictionary(self):
        scheme_dictionary = collections.defaultdict(dict)

        # Collect sections into buckets based on target name
        for scheme in self.schemes.values():
            sections_bucket = collections.defaultdict(list)

            for (sections_name, target_name) in scheme.entries:
                # Get the sections under the bucket 'target_name'. If this bucket does not exist
                # is is created automatically
                sections_in_bucket = sections_bucket[target_name]

                try:
                    sections = self.sections[sections_name]
                except KeyError:
                    message = GenerationException.UNDEFINED_REFERENCE + " to sections '" + sections + "'."
                    raise GenerationException(message, scheme)

                sections_in_bucket.append(sections)

            scheme_dictionary[scheme.name] = sections_bucket

        # Search for and raise exception on first instance of sections mapped to multiple targets
        for (scheme_name, sections_bucket) in scheme_dictionary.items():
            for sections_a, sections_b in itertools.combinations(sections_bucket.values(), 2):
                set_a = set()
                set_b = set()

                for sections in sections_a:
                    set_a.update(sections.entries)

                for sections in sections_b:
                    set_b.update(sections.entries)

                intersection = set_a.intersection(set_b)

                # If the intersection is a non-empty set, it means sections are mapped to multiple
                # targets. Raise exception.
                if intersection:
                    scheme = self.schemes[scheme_name]
                    message = "Sections " + str(intersection) + " mapped to multiple targets."
                    raise GenerationException(message, scheme)

        return scheme_dictionary

    def generate_rules(self, sdkconfig, sections_infos):
        placement_rules = collections.defaultdict(list)

        scheme_dictionary = self._build_scheme_dictionary()

        # Generate default rules
        default_rules = list()
        self._add_mapping_rules(None, None, None, GenerationModel.DEFAULT_SCHEME, scheme_dictionary, default_rules)

        all_mapping_rules = collections.defaultdict(list)

        # Generate rules based on mapping fragments
        for mapping in self.mappings.values():
            for (condition, entries) in mapping.entries:
                condition_true = False

                # Only non-default condition are evaluated agains sdkconfig model
                if condition != Mapping.DEFAULT_CONDITION:
                    try:
                        condition_true = sdkconfig.evaluate_expression(condition)
                    except Exception as e:
                        raise GenerationException(e.message, mapping)
                else:
                    condition_true = True

                if condition_true:

                    mapping_rules = list()

                    archive = mapping.archive
                    for (obj, symbol, scheme_name) in entries:
                        try:
                            self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
                        except KeyError:
                            message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
                            raise GenerationException(message, mapping)

                    all_mapping_rules[mapping.name] = mapping_rules

                    break   # Exit on first condition that evaluates to true

        # Detect rule conflicts
        for mapping_rules in all_mapping_rules.items():
            self._detect_conflicts(mapping_rules)

        # Add exclusions
        for mapping_rules in all_mapping_rules.values():
            self._create_exclusions(mapping_rules, default_rules, sections_infos)

        # Add the default rules grouped by target
        for default_rule in default_rules:
            existing_rules = placement_rules[default_rule.target]
            if default_rule.get_section_names():
                existing_rules.append(default_rule)

        for mapping_rules in all_mapping_rules.values():
            # Add the mapping rules grouped by target
            for mapping_rule in mapping_rules:
                existing_rules = placement_rules[mapping_rule.target]
                if mapping_rule.get_section_names():
                    existing_rules.append(mapping_rule)

        return placement_rules

    def _detect_conflicts(self, rules):
        (archive, rules_list) = rules

        for specificity in range(0, PlacementRule.OBJECT_SPECIFICITY + 1):
            rules_with_specificity = filter(lambda r: r.specificity == specificity, rules_list)

            for rule_a, rule_b in itertools.combinations(rules_with_specificity, 2):
                intersections = rule_a.get_sections_intersection(rule_b)

                if intersections and rule_a.maps_same_entities_as(rule_b):
                    rules_string = str([str(rule_a), str(rule_b)])
                    message = "Rules " + rules_string + " map sections " + str(list(intersections)) + " into multiple targets."
                    mapping = self.mappings[Mapping.get_mapping_name_from_archive(archive)]
                    raise GenerationException(message, mapping)

    def _create_extra_rules(self, rules):
        # This function generates extra rules for symbol specific rules. The reason for generating extra rules is to isolate,
        # as much as possible, rules that require expansion. Particularly, object specific extra rules are generated.
        rules_to_process = sorted(rules, key = lambda r: r.specificity)
        symbol_specific_rules = list(filter(lambda r: r.specificity == PlacementRule.SYMBOL_SPECIFICITY, rules_to_process))

        extra_rules = dict()

        for symbol_specific_rule in symbol_specific_rules:
            extra_rule_candidate = {s: None for s in symbol_specific_rule.get_section_names()}

            super_rules = filter(lambda r: symbol_specific_rule.is_more_specific_rule_of(r), rules_to_process)

            # Take a look at the existing rules that are more general than the current symbol-specific rule.
            # Only generate an extra rule if there is no existing object specific rule for that section
            for super_rule in super_rules:
                intersections = symbol_specific_rule.get_sections_intersection(super_rule)
                for intersection in intersections:
                    if super_rule.specificity != PlacementRule.OBJECT_SPECIFICITY:
                        extra_rule_candidate[intersection] = super_rule
                    else:
                        extra_rule_candidate[intersection] = None

            # Generate the extra rules for the symbol specific rule section, keeping track of the generated extra rules
            for (section, section_rule) in extra_rule_candidate.items():
                if section_rule:
                    extra_rule = None
                    extra_rules_key = (symbol_specific_rule.archive, symbol_specific_rule.obj, section_rule.target)

                    try:
                        extra_rule = extra_rules[extra_rules_key]

                        if section not in extra_rule.get_section_names():
                            new_rule = PlacementRule(extra_rule.archive, extra_rule.obj, extra_rule.symbol, list(extra_rule.get_section_names()) + [section] , extra_rule.target)
                            extra_rules[extra_rules_key] = new_rule
                    except KeyError:
                        extra_rule = PlacementRule(symbol_specific_rule.archive, symbol_specific_rule.obj, None, [section], section_rule.target)
                        extra_rules[extra_rules_key] = extra_rule

        return extra_rules.values()

    def _create_exclusions(self, mapping_rules, default_rules, sections_info):
        rules = list(default_rules)
        rules.extend(mapping_rules)

        extra_rules = self._create_extra_rules(rules)

        mapping_rules.extend(extra_rules)
        rules.extend(extra_rules)

        # Sort the rules by means of how specific they are. Sort by specificity from lowest to highest
        # * -> lib:* -> lib:obj -> lib:obj:symbol
        sorted_rules = sorted(rules, key = lambda r: r.specificity)

        # Now that the rules have been sorted, loop through each rule, and then loop
        # through rules below it (higher indeces), adding exclusions whenever appropriate.
        for general_rule in sorted_rules:
            for specific_rule in reversed(sorted_rules):
                if (specific_rule.specificity > general_rule.specificity and \
                    specific_rule.specificity != PlacementRule.SYMBOL_SPECIFICITY) or \
                    (specific_rule.specificity == PlacementRule.SYMBOL_SPECIFICITY and \
                    general_rule.specificity == PlacementRule.OBJECT_SPECIFICITY):
                    general_rule.add_exclusion(specific_rule, sections_info)

    def add_fragments_from_file(self, fragment_file):
        for fragment in fragment_file.fragments:
            dict_to_append_to = None

            if isinstance(fragment, Scheme):
                dict_to_append_to = self.schemes
            elif isinstance(fragment, Sections):
                dict_to_append_to = self.sections
            else:
                dict_to_append_to = self.mappings

            # Raise exception when the fragment of the same type is already in the stored fragments
            if fragment.name in dict_to_append_to.keys():
                stored = dict_to_append_to[fragment.name].path
                new = fragment.path
                message = "Duplicate definition of fragment '%s' found in %s and %s." % (fragment.name, stored, new)
                raise GenerationException(message)

            dict_to_append_to[fragment.name] = fragment

"""
Encapsulates a linker script template file. Finds marker syntax and handles replacement to generate the
final output.
"""
class TemplateModel:

    Marker = collections.namedtuple("Marker", "target indent rules")

    def __init__(self, template_file):
        self.members = []
        self.file = os.path.realpath(template_file.name)

        self._generate_members(template_file)

    def _generate_members(self, template_file):
        lines = template_file.readlines()

        target = Fragment.IDENTIFIER
        reference = Suppress("mapping") + Suppress("[") + target.setResultsName("target") + Suppress("]")
        pattern = White(" \t").setResultsName("indent") + reference

        # Find the markers in the template file line by line. If line does not match marker grammar,
        # set it as a literal to be copied as is to the output file.
        for line in lines:
            try:
                parsed = pattern.parseString(line)

                indent = parsed.indent
                target = parsed.target

                marker = TemplateModel.Marker(target, indent, [])

                self.members.append(marker)
            except ParseException:
                # Does not match marker syntax
                self.members.append(line)

    def fill(self, mapping_rules, sdkconfig):
        for member in self.members:
            target = None
            try:
                target = member.target
                indent = member.indent
                rules = member.rules

                del rules[:]

                rules.extend(mapping_rules[target])
            except KeyError:
                message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'."
                raise GenerationException(message)
            except AttributeError as a:
                pass

    def write(self, output_file):
        # Add information that this is a generated file.
        output_file.write("/* Automatically generated file; DO NOT EDIT */\n")
        output_file.write("/* Espressif IoT Development Framework Linker Script */\n")
        output_file.write("/* Generated from: %s */\n" % self.file)
        output_file.write("\n")

        # Do the text replacement
        for member in self.members:
            try:
                indent = member.indent
                rules = member.rules

                for rule in rules:
                    generated_line = "".join([indent, str(rule), '\n'])
                    output_file.write(generated_line)
            except AttributeError:
                output_file.write(member)

"""
Exception for linker script generation failures such as undefined references/ failure to
evaluate conditions, duplicate mappings, etc.
"""
class GenerationException(LdGenFailure):

    UNDEFINED_REFERENCE = "Undefined reference"

    def __init__(self, message, fragment=None):
        self.fragment = fragment
        self.message = message

    def __str__(self):
        if self.fragment:
            return "%s\nIn fragment '%s' defined in '%s'." % (self.message, self.fragment.name, self.fragment.path)
        else:
            return self.message

"""
Encapsulates an output of objdump. Contains information about the static library sections
and names
"""
class SectionsInfo(dict):

    __info = collections.namedtuple("__info", "filename content")

    def __init__(self):
        self.sections = dict()

    def add_sections_info(self, sections_info_file):
        first_line = sections_info_file.readline()

        archive_path = Literal("In archive").suppress() + Regex(r"[^:]+").setResultsName("archive_path") + Literal(":").suppress()
        parser = archive_path

        results = None

        try:
            results = parser.parseString(first_line)
        except ParseException as p:
            raise ParseException("File " + sections_info_file.name + " is not a valid sections info file. " + p.message)

        archive = os.path.basename(results.archive_path)
        self.sections[archive] = SectionsInfo.__info(sections_info_file.name, sections_info_file.read())

    def _get_infos_from_file(self, info):
        # Object file line: '{object}:  file format elf32-xtensa-le'
        object = Fragment.ENTITY.setResultsName("object") + Literal(":").suppress() + Literal("file format elf32-xtensa-le").suppress()

        # Sections table
        header = Suppress(Literal("Sections:") + Literal("Idx") + Literal("Name") + Literal("Size") + Literal("VMA") + Literal("LMA") + Literal("File off") + Literal("Algn"))
        entry = Word(nums).suppress() + Fragment.ENTITY + Suppress(OneOrMore(Word(alphanums, exact=8)) + Word(nums + "*") + ZeroOrMore(Word(alphas.upper()) + Optional(Literal(","))))

        # Content is object file line + sections table
        content = Group(object + header + Group(ZeroOrMore(entry)).setResultsName("sections"))

        parser = Group(ZeroOrMore(content)).setResultsName("contents")

        sections_info_text = info.content
        results = None

        try:
            results = parser.parseString(sections_info_text)
        except ParseException as p:
            raise ParseException("Unable to parse section info file " + info.filename + ". " + p.message)

        return results

    def get_obj_sections(self, archive, obj):
        stored = self.sections[archive]

        # Parse the contents of the sections file
        if not isinstance(stored, dict):
            parsed = self._get_infos_from_file(stored)
            stored = dict()
            for content in parsed.contents:
                sections = list(map(lambda s: s, content.sections))
                stored[content.object] = sections
            self.sections[archive] = stored

        for obj_key in stored.keys():
            if obj_key == obj + ".o" or obj_key == obj + ".c.obj":
                return stored[obj_key]