From a538644560b60d7dcbcd94aa428ffa05701f758c Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 12 Jan 2018 13:50:19 +1100 Subject: [PATCH 001/112] config: Add new Python & kconfiglib-based config management tool --- tools/kconfig_new/confgen.py | 150 + tools/kconfig_new/gen_kconfig_doc.py | 127 + tools/kconfig_new/kconfiglib.py | 4317 ++++++++++++++++++++++++++ 3 files changed, 4594 insertions(+) create mode 100755 tools/kconfig_new/confgen.py create mode 100644 tools/kconfig_new/gen_kconfig_doc.py create mode 100644 tools/kconfig_new/kconfiglib.py diff --git a/tools/kconfig_new/confgen.py b/tools/kconfig_new/confgen.py new file mode 100755 index 000000000..afca4176d --- /dev/null +++ b/tools/kconfig_new/confgen.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# +# Command line tool to take in ESP-IDF sdkconfig files with project +# settings and output data in multiple formats (update config, generate +# header file, generate .cmake include file, documentation, etc). +# +# Used internally by the ESP-IDF build system. But designed to be +# non-IDF-specific. +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import sys +import os +import os.path +import tempfile + +import gen_kconfig_doc +import kconfiglib + +__version__ = "0.1" + +def main(): + parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog='conftool') + + parser.add_argument('--config', + help='Project configuration settings', + nargs='?', + default=None) + + parser.add_argument('--kconfig', + help='KConfig file with config item definitions', + required=True) + + parser.add_argument('--output', nargs=2, action='append', + help='Write output file (format and output filename)', + metavar=('FORMAT', 'FILENAME')) + + parser.add_argument('--env', action='append', default=[], + help='Environment to set when evaluating the config file', metavar='NAME=VAL') + + args = parser.parse_args() + + for fmt, filename in args.output: + if not fmt in OUTPUT_FORMATS.keys(): + print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS)) + sys.exit(1) + + try: + args.env = [ (name,value) for (name,value) in ( e.split("=",1) for e in args.env) ] + except ValueError: + print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") + sys.exit(1) + + for name, value in args.env: + os.environ[name] = value + + config = kconfiglib.Kconfig(args.kconfig) + + if args.config is not None: + config.load_config(args.config) + + for output_type, filename in args.output: + temp_file = tempfile.mktemp(prefix="confgen_tmp") + try: + output_function = OUTPUT_FORMATS[output_type] + output_function(config, temp_file) + update_if_changed(temp_file, filename) + finally: + try: + os.remove(temp_file) + except OSError: + pass + + +def write_config(config, filename): + CONFIG_HEADING = """# +# Automatically generated file. DO NOT EDIT. +# Espressif IoT Development Framework (ESP-IDF) Project Configuration +# +""" + config.write_config(filename, header=CONFIG_HEADING) + +def write_header(config, filename): + CONFIG_HEADING = """/* + * Automatically generated file. DO NOT EDIT. + * Espressif IoT Development Framework (ESP-IDF) Configuration Header + */ +#pragma once +""" + config.write_autoconf(filename, header=CONFIG_HEADING) + +def write_cmake(config, filename): + with open(filename, "w") as f: + write = f.write + prefix = config.config_prefix + + write("""# +# Automatically generated file. DO NOT EDIT. +# Espressif IoT Development Framework (ESP-IDF) Configuration cmake include file +# +""") + def write_node(node): + sym = node.item + if not isinstance(sym, kconfiglib.Symbol): + return + + # Note: str_value calculates _write_to_conf, due to + # internal magic in kconfiglib... + val = sym.str_value + if sym._write_to_conf: + if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == "n": + val = "" # write unset values as empty variables + write("set({}{} \"{}\")\n".format( + prefix, sym.name, val)) + config.walk_menu(write_node) + + +def update_if_changed(source, destination): + with open(source, "r") as f: + source_contents = f.read() + if os.path.exists(destination): + with open(destination, "r") as f: + dest_contents = f.read() + if source_contents == dest_contents: + return # nothing to update + + with open(destination, "w") as f: + f.write(source_contents) + + +OUTPUT_FORMATS = { + "config" : write_config, + "header" : write_header, + "cmake" : write_cmake, + "docs" : gen_kconfig_doc.write_docs } + +if __name__ == '__main__': + main() diff --git a/tools/kconfig_new/gen_kconfig_doc.py b/tools/kconfig_new/gen_kconfig_doc.py new file mode 100644 index 000000000..6614c54c3 --- /dev/null +++ b/tools/kconfig_new/gen_kconfig_doc.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# gen_kconfig_doc - confgen.py support for generating ReST markup documentation +# +# For each option in the loaded Kconfig (e.g. 'FOO'), CONFIG_FOO link target is +# generated, allowing options to be referenced in other documents +# (using :ref:`CONFIG_FOO`) +# +# Copyright 2017 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import kconfiglib + +# Indentation to be used in the generated file +INDENT = ' ' + +# Characters used when underlining section heading +HEADING_SYMBOLS = '#*=-^"+' + +# Keep the heading level in sync with api-reference/kconfig.rst +INITIAL_HEADING_LEVEL = 2 +MAX_HEADING_LEVEL = 5 + +def write_docs(config, filename): + """ Note: writing .rst documentation ignores the current value + of any items. ie the --config option can be ignored. """ + with open(filename, "w") as f: + config.walk_menu(lambda node: write_menu_item(f, node)) + + +def get_breadcrumbs(node): + # this is a bit wasteful as it recalculates each time, but still... + result = [] + node = node.parent + while node.parent: + if node.prompt: + result = [ node.prompt[0] ] + result + node = node.parent + return " > ".join(result) + +def get_heading_level(node): + # bit wasteful also + result = INITIAL_HEADING_LEVEL + node = node.parent + while node.parent: + result += 1 + if result == MAX_HEADING_LEVEL: + return MAX_HEADING_LEVEL + node = node.parent + return result + +def write_menu_item(f, node): + if not node.prompt: + return # Don't do anything for invisible menu items + + if isinstance(node.parent.item, kconfiglib.Choice): + return # Skip choice nodes, they are handled as part of the parent (see below) + + try: + name = node.item.name + except AttributeError: + name = None + + try: + is_menu = node.item == kconfiglib.MENU or node.is_menuconfig + except AttributeError: + is_menu = False # not all MenuNodes have is_menuconfig for some reason + + ## Heading + if name: + title = name + # add link target so we can use :ref:`CONFIG_FOO` + f.write('.. _CONFIG_%s:\n\n' % name) + else: + title = node.prompt[0] + + # if no symbol name, use the prompt as the heading + if is_menu: + f.write('%s\n' % title) + f.write(HEADING_SYMBOLS[get_heading_level(node)] * len(title)) + f.write('\n\n') + else: + f.write('**%s**\n\n\n % title') + + if name: + f.write('%s%s\n\n' % (INDENT, node.prompt[0])) + f.write('%s:emphasis:`Found in: %s`\n\n' % (INDENT, get_breadcrumbs(node))) + + try: + if node.help: + # Help text normally contains newlines, but spaces at the beginning of + # each line are stripped by kconfiglib. We need to re-indent the text + # to produce valid ReST. + f.write('%s%s\n' % (INDENT, node.help.replace('\n', '\n%s' % INDENT))) + except AttributeError: + pass # No help + + if isinstance(node.item, kconfiglib.Choice): + f.write('%sAvailable options:\n' % INDENT) + choice_node = node.list + while choice_node: + # Format available options as a list + f.write('%s- %-20s (%s)\n' % (INDENT * 2, choice_node.prompt[0], choice_node.item.name)) + if choice_node.help: + HELP_INDENT = INDENT * 2 + fmt_help = choice_node.help.replace('\n', '\n%s ' % HELP_INDENT) + f.write('%s \n%s %s\n' % (HELP_INDENT, HELP_INDENT, fmt_help)) + choice_node = choice_node.next + + f.write('\n\n') + + +if __name__ == '__main__': + print("Run this via 'confgen.py --output doc FILENAME'") + diff --git a/tools/kconfig_new/kconfiglib.py b/tools/kconfig_new/kconfiglib.py new file mode 100644 index 000000000..527c5a55a --- /dev/null +++ b/tools/kconfig_new/kconfiglib.py @@ -0,0 +1,4317 @@ +# Copyright (c) 2011-2017, Ulf Magnusson +# Modifications (c) 2018 Espressif Systems +# SPDX-License-Identifier: ISC +# +# ******* IMPORTANT ********** +# +# This is kconfiglib 2.1.0 with some modifications to match the behaviour +# of the ESP-IDF kconfig: +# +# - 'source' nows uses wordexp(3) behaviour to allow source-ing multiple +# files at once, and to expand environment variables directly in the source +# command (without them having to be set as properties in the Kconfig file) +# +# - Added walk_menu() function and refactored to use this internally. +# +# - BOOL & TRISTATE items are allowed to have blank values in .config +# (equivalent to n, this is backwards compatibility with old IDF conf.c) +# +""" +Overview +======== + +Kconfiglib is a Python 2/3 library for scripting and extracting information +from Kconfig configuration systems. It can be used for the following, among +other things: + + - Programmatically get and set symbol values + + allnoconfig.py and allyesconfig.py examples are provided, automatically + verified to produce identical output to the standard 'make allnoconfig' and + 'make allyesconfig'. + + - Read and write .config files + + The generated .config files are character-for-character identical to what + the C implementation would generate (except for the header comment). The + test suite relies on this, as it compares the generated files. + + - Inspect symbols + + Printing a symbol gives output which could be fed back into a Kconfig parser + to redefine it***. The printing function (__str__()) is implemented with + public APIs, meaning you can fetch just whatever information you need as + well. + + A helpful __repr__() is implemented on all objects too, also implemented + with public APIs. + + ***Choice symbols get their parent choice as a dependency, which shows up as + e.g. 'prompt "choice symbol" if ' when printing the symbol. This + could easily be worked around if 100% reparsable output is needed. + + - Inspect expressions + + Expressions use a simple tuple-based format that can be processed manually + if needed. Expression printing and evaluation functions are provided, + implemented with public APIs. + + - Inspect the menu tree + + The underlying menu tree is exposed, including submenus created implicitly + from symbols depending on preceding symbols. This can be used e.g. to + implement menuconfig-like functionality. See the menuconfig.py example. + + +Here are some other features: + + - Single-file implementation + + The entire library is contained in this file. + + - Runs unmodified under both Python 2 and Python 3 + + The code mostly uses basic Python features and has no third-party + dependencies. The most advanced things used are probably @property and + __slots__. + + - Robust and highly compatible with the standard Kconfig C tools + + The test suite automatically compares output from Kconfiglib and the C tools + by diffing the generated .config files for the real kernel Kconfig and + defconfig files, for all ARCHes. + + This currently involves comparing the output for 36 ARCHes and 498 defconfig + files (or over 18000 ARCH/defconfig combinations in "obsessive" test suite + mode). All tests are expected to pass. + + - Not horribly slow despite being a pure Python implementation + + The allyesconfig.py example currently runs in about 1.6 seconds on a Core i7 + 2600K (with a warm file cache), where half a second is overhead from 'make + scriptconfig' (see below). + + For long-running jobs, PyPy gives a big performance boost. CPython is faster + for short-running jobs as PyPy needs some time to warm up. + + - Internals that (mostly) mirror the C implementation + + While being simpler to understand. + + +Using Kconfiglib on the Linux kernel with the Makefile targets +============================================================== + +For the Linux kernel, a handy interface is provided by the +scripts/kconfig/Makefile patch. Apply it with either 'git am' or the 'patch' +utility: + + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | git am + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | patch -p1 + +Warning: Not passing -p1 to patch will cause the wrong file to be patched. + +Please tell me if the patch does not apply. It should be trivial to apply +manually, as it's just a block of text that needs to be inserted near the other +*conf: targets in scripts/kconfig/Makefile. + +If you do not wish to install Kconfiglib via pip, the Makefile patch is set up +so that you can also just clone Kconfiglib into the kernel root: + + $ git clone git://github.com/ulfalizer/Kconfiglib.git + $ git am Kconfiglib/makefile.patch (or 'patch -p1 < Kconfiglib/makefile.patch') + +Warning: The directory name Kconfiglib/ is significant in this case, because +it's added to PYTHONPATH by the new targets in makefile.patch. + +Look further down for a motivation for the Makefile patch and for instructions +on how you can use Kconfiglib without it. + +The Makefile patch adds the following targets: + + +make [ARCH=] iscriptconfig +-------------------------------- + +This target gives an interactive Python prompt where a Kconfig instance has +been preloaded and is available in 'kconf'. To change the Python interpreter +used, pass PYTHONCMD= to make. The default is "python". + +To get a feel for the API, try evaluating and printing the symbols in +kconf.defined_syms, and explore the MenuNode menu tree starting at +kconf.top_node by following 'next' and 'list' pointers. + +The item contained in a menu node is found in MenuNode.item (note that this can +be one of the constants MENU and COMMENT), and all symbols and choices have a +'nodes' attribute containing their menu nodes (usually only one). Printing a +menu node will print its item, in Kconfig format. + +If you want to look up a symbol by name, use the kconf.syms dictionary. + + +make scriptconfig SCRIPT=