#!/usr/bin/env python # # Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import re import collections import itertools import os import subprocess import fnmatch from sdkconfig import SDKConfig from fragments import FragmentFileModel, Sections, Scheme, Mapping, Fragment from pyparsing import * """ Encapsulates a generated placement rule placed under a target """ class PlacementRule(): DEFAULT_SPECIFICITY = 0 ARCHIVE_SPECIFICITY = 1 OBJECT_SPECIFICITY = 2 SYMBOL_SPECIFICITY = 3 class __container(): def __init__(self, content): self.content = content __metadata = collections.namedtuple("__metadata", "excludes expansions expanded") def __init__(self, archive, obj, symbol, sections, target): if archive == "*": archive = None if obj == "*": obj = None self.archive = archive self.obj = obj self.symbol = symbol self.target = target self.sections = dict() self.specificity = 0 self.specificity += 1 if self.archive else 0 self.specificity += 1 if (self.obj and not self.obj == '*') else 0 self.specificity += 1 if self.symbol else 0 for section in sections: section_data = Sections.get_section_data_from_entry(section, self.symbol) if not self.symbol: for s in section_data: metadata = self.__metadata(self.__container([]), self.__container([]), self.__container(False)) self.sections[s] = metadata else: (section, expansion) = section_data if expansion: metadata = self.__metadata(self.__container([]), self.__container([expansion]), self.__container(True)) self.sections[section] = metadata def get_section_names(self): return self.sections.keys() def add_exclusion(self, other, sections_infos = None): # Utility functions for this method def do_section_expansion(rule, section): if section in rule.get_section_names(): sections_in_obj = sections_infos.get_obj_sections(rule.archive, rule.obj) expansions = fnmatch.filter(sections_in_obj, section) return expansions def remove_section_expansions(rule, section, expansions): existing_expansions = self.sections[section].expansions.content self.sections[section].expansions.content = [e for e in existing_expansions if e not in expansions] # Exit immediately if the exclusion to be added is more general than this rule. if not other.is_more_specific_rule_of(self): return for section in self.get_sections_intersection(other): if(other.specificity == PlacementRule.SYMBOL_SPECIFICITY): # If this sections has not been expanded previously, expand now and keep track. previously_expanded = self.sections[section].expanded.content if not previously_expanded: expansions = do_section_expansion(self, section) if expansions: self.sections[section].expansions.content = expansions self.sections[section].expanded.content = True previously_expanded = True # Remove the sections corresponding to the symbol name remove_section_expansions(self, section, other.sections[section].expansions.content) # If it has been expanded previously but now the expansions list is empty, # it means adding exclusions has exhausted the list. Remove the section entirely. if previously_expanded and not self.sections[section].expanded.content: del self.sections[section] else: # A rule section can have multiple rule sections excluded from it. Get the # most specific rule from the list, and if an even more specific rule is found, # replace it entirely. Otherwise, keep appending. exclusions = self.sections[section].excludes exclusions_list = exclusions.content if exclusions.content != None else [] exclusions_to_remove = filter(lambda r: r.is_more_specific_rule_of(other), exclusions_list) remaining_exclusions = [e for e in exclusions_list if e not in exclusions_to_remove] remaining_exclusions.append(other) self.sections[section].excludes.content = remaining_exclusions def get_sections_intersection(self, other): return set(self.sections.keys()).intersection(set(other.sections.keys())) def is_more_specific_rule_of(self, other): if (self.specificity <= other.specificity): return False # Compare archive, obj and target for entity_index in range (1, other.specificity + 1): if self[entity_index] != other[entity_index] and other[entity_index] != None: return False return True def maps_same_entities_as(self, other): if self.specificity != other.specificity: return False # Compare archive, obj and target for entity_index in range (1, other.specificity + 1): if self[entity_index] != other[entity_index] and other[entity_index] != None: return False return True def __getitem__(self, key): if key == PlacementRule.ARCHIVE_SPECIFICITY: return self.archive elif key == PlacementRule.OBJECT_SPECIFICITY: return self.obj elif key == PlacementRule.SYMBOL_SPECIFICITY: return self.symbol else: return None def __str__(self): sorted_sections = sorted(self.get_section_names()) sections_string = list() for section in sorted_sections: exclusions = self.sections[section].excludes.content exclusion_string = None if exclusions: exclusion_string = " ".join(map(lambda e: "*" + e.archive + (":" + e.obj + ".*" if e.obj else ""), exclusions)) exclusion_string = "EXCLUDE_FILE(" + exclusion_string + ")" else: exclusion_string = "" section_string = None exclusion_section_string = None section_expansions = self.sections[section].expansions.content section_expanded = self.sections[section].expanded.content if section_expansions and section_expanded: section_string = " ".join(section_expansions) exclusion_section_string = section_string else: section_string = section exclusion_section_string = exclusion_string + " " + section_string sections_string.append(exclusion_section_string) sections_string = " ".join(sections_string) archive = str(self.archive) if self.archive else "" obj = (str(self.obj) + (".*" if self.obj else "")) if self.obj else "" # Handle output string generation based on information available if self.specificity == PlacementRule.DEFAULT_SPECIFICITY: rule_string = "*(%s)" % (sections_string) elif self.specificity == PlacementRule.ARCHIVE_SPECIFICITY: rule_string = "*%s:(%s)" % (archive, sections_string) else: rule_string = "*%s:%s(%s)" % (archive, obj, sections_string) return rule_string def __eq__(self, other): if id(self) == id(other): return True def exclusions_set(exclusions): exclusions_set = {(e.archive, e.obj, e.symbol, e.target) for e in exclusions} return exclusions_set if self.archive != other.archive: return False if self.obj != other.obj: return False if self.symbol != other.symbol: return False if set(self.sections.keys()) != set(other.sections.keys()): return False for (section, metadata) in self.sections.items(): self_meta = metadata other_meta = other.sections[section] if exclusions_set(self_meta.excludes.content) != exclusions_set(other_meta.excludes.content): return False if set(self_meta.expansions.content) != set(other_meta.expansions.content): return False return True def __ne__(self, other): return not self.__eq__(other) def __iter__(self): yield self.archive yield self.obj yield self.symbol raise StopIteration """ Implements generation of placement rules based on collected sections, scheme and mapping fragment. """ class GenerationModel: DEFAULT_SCHEME = "default" def __init__(self): self.schemes = {} self.sections = {} self.mappings = {} def _add_mapping_rules(self, archive, obj, symbol, scheme_name, scheme_dict, rules): # Use an ordinary dictionary to raise exception on non-existing keys temp_dict = dict(scheme_dict) sections_bucket = temp_dict[scheme_name] for (target, sections) in sections_bucket.items(): section_entries = [] for section in sections: section_entries.extend(section.entries) rule = PlacementRule(archive, obj, symbol, section_entries, target) if not rule in rules: rules.append(rule) def _build_scheme_dictionary(self): scheme_dictionary = collections.defaultdict(dict) # Collect sections into buckets based on target name for scheme in self.schemes.values(): sections_bucket = collections.defaultdict(list) for (sections_name, target_name) in scheme.entries: # Get the sections under the bucket 'target_name'. If this bucket does not exist # is is created automatically sections_in_bucket = sections_bucket[target_name] try: sections = self.sections[sections_name] except KeyError: message = GenerationException.UNDEFINED_REFERENCE + " to sections '" + sections + "'." raise GenerationException(message, scheme) sections_in_bucket.append(sections) scheme_dictionary[scheme.name] = sections_bucket # Search for and raise exception on first instance of sections mapped to multiple targets for (scheme_name, sections_bucket) in scheme_dictionary.items(): for sections_a, sections_b in itertools.combinations(sections_bucket.values(), 2): set_a = set() set_b = set() for sections in sections_a: set_a.update(sections.entries) for sections in sections_b: set_b.update(sections.entries) intersection = set_a.intersection(set_b) # If the intersection is a non-empty set, it means sections are mapped to multiple # targets. Raise exception. if intersection: scheme = self.schemes[scheme_name] message = "Sections " + str(intersection) + " mapped to multiple targets." raise GenerationException(message, scheme) return scheme_dictionary def generate_rules(self, sdkconfig, sections_infos): placement_rules = collections.defaultdict(list) scheme_dictionary = self._build_scheme_dictionary() # Generate default rules default_rules = list() self._add_mapping_rules(None, None, None, GenerationModel.DEFAULT_SCHEME, scheme_dictionary, default_rules) all_mapping_rules = collections.defaultdict(list) # Generate rules based on mapping fragments for mapping in self.mappings.values(): for (condition, entries) in mapping.entries: condition_true = False # Only non-default condition are evaluated agains sdkconfig model if condition != Mapping.DEFAULT_CONDITION: try: condition_true = sdkconfig.evaluate_expression(condition) except Exception as e: raise GenerationException(e.message, mapping) else: condition_true = True if condition_true: mapping_rules = list() archive = mapping.archive for (obj, symbol, scheme_name) in entries: try: self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules) except KeyError: message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'." raise GenerationException(message, mapping) all_mapping_rules[mapping.name] = mapping_rules break # Exit on first condition that evaluates to true # Detect rule conflicts for mapping_rules in all_mapping_rules.items(): self._detect_conflicts(mapping_rules) # Add exclusions for mapping_rules in all_mapping_rules.values(): self._create_exclusions(mapping_rules, default_rules, sections_infos) # Add the default rules grouped by target for default_rule in default_rules: existing_rules = placement_rules[default_rule.target] if default_rule.get_section_names(): existing_rules.append(default_rule) for mapping_rules in all_mapping_rules.values(): # Add the mapping rules grouped by target for mapping_rule in mapping_rules: existing_rules = placement_rules[mapping_rule.target] if mapping_rule.get_section_names(): existing_rules.append(mapping_rule) return placement_rules def _detect_conflicts(self, rules): (archive, rules_list) = rules for specificity in range(0, PlacementRule.OBJECT_SPECIFICITY + 1): rules_with_specificity = filter(lambda r: r.specificity == specificity, rules_list) for rule_a, rule_b in itertools.combinations(rules_with_specificity, 2): intersections = rule_a.get_sections_intersection(rule_b) if intersections and rule_a.maps_same_entities_as(rule_b): rules_string = str([str(rule_a), str(rule_b)]) message = "Rules " + rules_string + " map sections " + str(list(intersections)) + " into multiple targets." mapping = self.mappings[Mapping.get_mapping_name_from_archive(archive)] raise GenerationException(message, mapping) def _create_extra_rules(self, rules): # This function generates extra rules for symbol specific rules. The reason for generating extra rules is to isolate, # as much as possible, rules that require expansion. Particularly, object specific extra rules are generated. rules_to_process = sorted(rules, key = lambda r: r.specificity) symbol_specific_rules = list(filter(lambda r: r.specificity == PlacementRule.SYMBOL_SPECIFICITY, rules_to_process)) extra_rules = dict() for symbol_specific_rule in symbol_specific_rules: extra_rule_candidate = {s: None for s in symbol_specific_rule.get_section_names()} super_rules = filter(lambda r: symbol_specific_rule.is_more_specific_rule_of(r), rules_to_process) # Take a look at the existing rules that are more general than the current symbol-specific rule. # Only generate an extra rule if there is no existing object specific rule for that section for super_rule in super_rules: intersections = symbol_specific_rule.get_sections_intersection(super_rule) for intersection in intersections: if super_rule.specificity != PlacementRule.OBJECT_SPECIFICITY: extra_rule_candidate[intersection] = super_rule else: extra_rule_candidate[intersection] = None # Generate the extra rules for the symbol specific rule section, keeping track of the generated extra rules for (section, section_rule) in extra_rule_candidate.items(): if section_rule: extra_rule = None extra_rules_key = (symbol_specific_rule.archive, symbol_specific_rule.obj, section_rule.target) try: extra_rule = extra_rules[extra_rules_key] if section not in extra_rule.get_section_names(): new_rule = PlacementRule(extra_rule.archive, extra_rule.obj, extra_rule.symbol, list(extra_rule.get_section_names()) + [section] , extra_rule.target) extra_rules[extra_rules_key] = new_rule except KeyError: extra_rule = PlacementRule(symbol_specific_rule.archive, symbol_specific_rule.obj, None, [section], section_rule.target) extra_rules[extra_rules_key] = extra_rule return extra_rules.values() def _create_exclusions(self, mapping_rules, default_rules, sections_info): rules = list(default_rules) rules.extend(mapping_rules) extra_rules = self._create_extra_rules(rules) mapping_rules.extend(extra_rules) rules.extend(extra_rules) # Sort the rules by means of how specific they are. Sort by specificity from lowest to highest # * -> lib:* -> lib:obj -> lib:obj:symbol sorted_rules = sorted(rules, key = lambda r: r.specificity) # Now that the rules have been sorted, loop through each rule, and then loop # through rules below it (higher indeces), adding exclusions whenever appropriate. for general_rule in sorted_rules: for specific_rule in reversed(sorted_rules): if (specific_rule.specificity > general_rule.specificity and \ specific_rule.specificity != PlacementRule.SYMBOL_SPECIFICITY) or \ (specific_rule.specificity == PlacementRule.SYMBOL_SPECIFICITY and \ general_rule.specificity == PlacementRule.OBJECT_SPECIFICITY): general_rule.add_exclusion(specific_rule, sections_info) def add_fragments_from_file(self, fragment_file): for fragment in fragment_file.fragments: dict_to_append_to = None if isinstance(fragment, Scheme): dict_to_append_to = self.schemes elif isinstance(fragment, Sections): dict_to_append_to = self.sections else: dict_to_append_to = self.mappings # Raise exception when the fragment of the same type is already in the stored fragments if fragment.name in dict_to_append_to.keys(): stored = dict_to_append_to[fragment.name].path new = fragment.path message = "Duplicate definition of fragment '%s' found in %s and %s." % (fragment.name, stored, new) raise GenerationException(message) dict_to_append_to[fragment.name] = fragment """ Encapsulates a linker script template file. Finds marker syntax and handles replacement to generate the final output. """ class TemplateModel: Marker = collections.namedtuple("Marker", "target indent rules") def __init__(self, template_file): self.members = [] self.file = os.path.realpath(template_file.name) self._generate_members(template_file) def _generate_members(self, template_file): lines = template_file.readlines() target = Fragment.IDENTIFIER reference = Suppress("mapping") + Suppress("[") + target.setResultsName("target") + Suppress("]") pattern = White(" \t").setResultsName("indent") + reference # Find the markers in the template file line by line. If line does not match marker grammar, # set it as a literal to be copied as is to the output file. for line in lines: try: parsed = pattern.parseString(line) indent = parsed.indent target = parsed.target marker = TemplateModel.Marker(target, indent, []) self.members.append(marker) except ParseException: # Does not match marker syntax self.members.append(line) def fill(self, mapping_rules, sdkconfig): for member in self.members: target = None try: target = member.target indent = member.indent rules = member.rules del rules[:] rules.extend(mapping_rules[target]) except KeyError: message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'." raise GenerationException(message) except AttributeError as a: pass def write(self, output_file): # Add information that this is a generated file. output_file.write("/* Automatically generated file; DO NOT EDIT */\n") output_file.write("/* Espressif IoT Development Framework Linker Script */\n") output_file.write("/* Generated from: %s */\n" % self.file) output_file.write("\n") # Do the text replacement for member in self.members: try: indent = member.indent rules = member.rules for rule in rules: generated_line = "".join([indent, str(rule), '\n']) output_file.write(generated_line) except AttributeError: output_file.write(member) """ Exception for linker script generation failures such as undefined references/ failure to evaluate conditions, duplicate mappings, etc. """ class GenerationException(Exception): UNDEFINED_REFERENCE = "Undefined reference" def __init__(self, message, fragment=None): self.fragment = fragment self.message = message def __str__(self): if self.fragment: return "%s\nIn fragment '%s' defined in '%s'." % (self.message, self.fragment.name, self.fragment.path) else: return self.message """ Encapsulates an output of objdump. Contains information about the static library sections and names """ class SectionsInfo(dict): PATH = Optional("/") + ZeroOrMore(Regex(r"[^/.]+") + Literal("/")) __info = collections.namedtuple("__info", "filename content") def __init__(self): self.sections = dict() def add_sections_info(self, sections_info_file): first_line = sections_info_file.readline() archive = Literal("In archive").suppress() + SectionsInfo.PATH.suppress() + Fragment.ENTITY.setResultsName("archive") + Literal(":").suppress() parser = archive results = None try: results = parser.parseString(first_line) except ParseException as p: raise ParseException("File " + sections_info_file.name + " is not a valid sections info file. " + p.message) self.sections[results.archive] = SectionsInfo.__info(sections_info_file.name, sections_info_file.read()) def _get_infos_from_file(self, info): # Object file line: '{object}: file format elf32-xtensa-le' object = Fragment.ENTITY.setResultsName("object") + Literal(":").suppress() + Literal("file format elf32-xtensa-le").suppress() # Sections table header = Suppress(Literal("Sections:") + Literal("Idx") + Literal("Name") + Literal("Size") + Literal("VMA") + Literal("LMA") + Literal("File off") + Literal("Algn")) entry = Word(nums).suppress() + Fragment.ENTITY + Suppress(OneOrMore(Word(alphanums, exact=8)) + Word(nums + "*") + ZeroOrMore(Word(alphas.upper()) + Optional(Literal(",")))) # Content is object file line + sections table content = Group(object + header + Group(ZeroOrMore(entry)).setResultsName("sections")) parser = Group(ZeroOrMore(content)).setResultsName("contents") sections_info_text = info.content results = None try: results = parser.parseString(sections_info_text) except ParseException as p: raise ParseException("Unable to parse section info file " + info.filename + ". " + p.message) return results def get_obj_sections(self, archive, obj): stored = self.sections[archive] # Parse the contents of the sections file if not isinstance(stored, dict): parsed = self._get_infos_from_file(stored) stored = dict() for content in parsed.contents: sections = list(map(lambda s: s, content.sections)) stored[content.object] = sections self.sections[archive] = stored for obj_key in stored.keys(): if obj_key == obj + ".o" or obj_key == obj + ".c.obj": return stored[obj_key]